{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chaper 2 - N-armed Bandits\n",
    "### Deep Reinforcement Learning _in Action_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_action(actions):\n",
    "\tbest_action = 0\n",
    "\tmax_action_value = 0\n",
    "\tfor i in range(len(actions)): #A \n",
    "\t\tcur_action_value = get_action_value(actions[i]) #B\n",
    "\t\tif cur_action_value > max_action_value:\n",
    "\t\t\tbest_action = i\n",
    "\t\t\tmax_action_value = cur_action_value\n",
    "\treturn best_action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "n = 10\n",
    "probs = np.random.rand(n) #A\n",
    "eps = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_reward(prob, n=10):\n",
    "    reward = 0;\n",
    "    for i in range(n):\n",
    "        if random.random() < prob:\n",
    "            reward += 1\n",
    "    return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "reward_test = [get_reward(0.7) for _ in range(2000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.001"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(reward_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum = 0\n",
    "x = [4,5,6,7]\n",
    "for j in range(len(x)): \n",
    "\tsum = sum + x[j]\n",
    "sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAFKCAYAAAD/tU6fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xm0JWV97vHvIyACKmODhAYaBWcjYMugxAg4gBIhDjgiKIYbxUSjiaLeDF71inGelSsoTigiRiIOtEzGCWwEFUSkgyANSLcKqAFl8Hf/qDqwOfQ5feqcvXufLr+ftfbaVW/Vrv3b9FrN02+99b6pKiRJkvrmbuMuQJIkaRQMOZIkqZcMOZIkqZcMOZIkqZcMOZIkqZcMOZIkqZcMOZIkqZcMOZIkqZcMOZIkqZfWHXcBa8IWW2xRixYtGncZkiRpCM4777xfVtWC1Z33JxFyFi1axNKlS8ddhiRJGoIkV8zkPG9XSZKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXjLkSJKkXvqTWLtKkoZp0VGnjruEobj86CePuwRppOzJkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvWTIkSRJvTQvQk6Sy5P8KMkFSZa2bZslWZLk0vZ907Y9Sd6TZFmSHybZdbzVS5Kk+WhehJzW3lW1c1UtbvePAk6vqp2A09t9gP2BndrXEcAH13ilkiRp3ptPIWeyA4Hj2+3jgYMG2j9eje8CmyTZehwFSpKk+Wu+hJwCTktyXpIj2ratquqadvsXwFbt9jbAlQOfXd623UmSI5IsTbJ05cqVo6pbkiTNU+uOu4DWXlV1VZItgSVJfjJ4sKoqSXW5YFUdAxwDsHjx4k6flSRJa7950ZNTVVe17yuALwC7AddO3IZq31e0p18FbDvw8YVtmyRJ0u3GHnKSbJTkXhPbwBOAC4FTgEPb0w4FvthunwI8v33Kag/ghoHbWpIkScD8uF21FfCFJNDU8+mq+mqS7wEnJjkcuAI4uD3/y8CTgGXAjcAL1nzJkiRpvht7yKmqy4CHr6L9V8C+q2gv4Mg1UJokSVqLjf12lSRJ0igYciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi+NfcZjSdJ4LDrq1HGXMDSXH/3kcZegecieHEmS1EtDCTlJ7pnkEUm2HMb1JEmS5mrGISfJ3kk+kGSXSe2HAdcC5wJXJXnjcEuUJEnqrktPzouAFwKXTzQk2QE4BtgAuKptfk2Su6weLkmStCZ1CTm7AT+oqusG2g6hGbz86qraDtgTKOAlwytRkiSpuy4hZwGwfFLbPsDvgfcBVNVS4NvAw4dSnSRJ0ix1CTkbArdM7CS5G7AYOLeqbho470pg6+GUJ0mSNDtdQs4KYMeB/T1ogs+3Jp23PnATkiRJY9Ql5HwH2CXJwUnuDbyOZvzNkknnPQi4ekj1SZIkzUqXkPNW4FbgBOA6YH/g/Ko6a+KEJAtpQs7SIdYoSZLU2YxDTlWdCxwAnA1cDHwMmDyP9jOBG7hr744kSdIa1WntqqpawjQBpqreDrx9rkVJkiTNlWtXSZKkXjLkSJKkXuoUcpIsSvLhJMuS3Jjktilet46qYEmSpJmY8ZicJA8BvgncG8jqTp9LUZIkSXPVpSfnTcDGwFeA3YGNq+puU71GUq0kSdIMdXm66jE0K5D/dVXdsppzJUmSxqpLj8v6wPcMOJIkaW3QJeT8lOZ2lSRJ0rzXJeT8P+AxSRaNphRJkqTh6bKswweAE4GvJ9k/iYOLJUnSvNXlEfLL2s1FwJeAW5NcA/xxFadXVd1v7uVJkiTNTpenqxYNbAdYD9huinNrtgVJkiQNQ5eQs8PIqpAkSRqyGYecqrpilIVI6rdFR5067hIk/Ylx8LAkSeqlLrerbpdkT+CxwDZt01XAWVX1nSHVJUmSNCedQk47R86ngD0mmtr3ao9/B3heVV3etZAk6wBLgauq6oAkOwCfATYHzgMOqaqbk6wPfBx4BPAr4Jmz+T5JktRvM75dlWQz4ExgT+B/aALIm9rXZ9q2RwFnJNl0FrW8DLh4YP8twDurakfgOuDwtv1w4Lq2/Z3teZIkSXfSZUzOPwHbAycBO1TVc6vqn9vXc2keMT+pff+nLkUkWQg8GfhIux9gn/Z6AMcDB7XbB7b7tMf3bc+XJEm6XZeQcyBwDc1to19NPlhVvwYOac85aPLx1XgX8CrumFhwc+D6qrq13V/OHeN/tgGubL/zVuCG9nxJkqTbdQk5i4D/qqo/THVCe+y/uPPEgdNKcgCwoqrO61DLTK57RJKlSZauXLlymJeWJElrgS4h5xZgwxmct0F77kw9GnhKkstpxvbsA7wb2CTJxMDohTRPcNG+bwvQHt+YZgDynVTVMVW1uKoWL1iwoEM5kiSpD7qEnIuBvZPcZ6oT2mP7cOcBxNOqqtdU1cKqWgQ8CzijHeNzJvD09rRDgS+226e0+7THz6gql5GQJEl30iXkfBLYiGYV8n0mH0yyN3AaTW/PJ4ZQ26uBVyRZRjPm5ti2/Vhg87b9FcBRQ/guSZLUM13myfkQ8DTgL4ElSa4GfkYzR84ONAOCQ9MD86HZFFNVZwFntduXAbut4pzfA8+YzfUlSdKfjhn35LRPMu0HvI1mTpxtgL2Av6AZM/M/7bEnVdVtwy9VkiRp5jrNeNw+PfWqJP9CM+Pw4LIO57W9LJIkSWM3q7Wr2jDzrSHXIkmSNDSuQi5Jknppyp6cJI9pN8+tqt8P7M9IVX1jTpVJkiTNwXS3q86ieXLqQcBPB/ZnolZzbUmSpJGaLoh8gyas3DhpX5Ikad6bMuRU1WOn25ckSZrPHHgsSZJ6acYhJ8lxSV44g/MOS3Lc3MqSJEmamy49OYfRzHC8Oo/mjgU0JUmSxmIUt6vWA/44gutKkiTN2ChCzkOA60dwXUmSpBmbdi6bVYyt2Wua8Tbr0sypsytw6hBqkyRJmrXVTdh32MB2ATu2r+n8AnjdHGqSJEmas9WFnBe07wGOA74JHDvFuTfTrEb+3aq6eTjlSZIkzc60Iaeqjp/YTvJvNAHm+Kk/IUmSND/MeH2pqlo0wjokSZKGyhmPJUlSL3VeKTzJPYC9gfsD96YZrzNZVdUb5libJEnSrHUKOUmeBnwI2Gy602iexDLkSJKksZlxyEmyO/AZmtmMTwAeCjwMOJrmsfLHAxvTPH21fOiVSpIkddClJ+cfacbwHFRVpyb5KPCwqnodQJItgI8CT6KZEFCSJGlsugw8fhRwYVWtcjbjqvol8BxgfeD1Q6hNkiRp1rqEnC2ASwb2bwVIssFEQ1X9FvgGsP9QqpMkSZqlLiHnOppemgkTi3AunHReAVvOpShJkqS56hJyrgS2G9i/kOZJqgMmGpJsBOxFs7yDJEnS2HQZeHwW8LIkC6pqJfAl4EbgzUnuQ/NE1fNpbmudPOxCJUmSuugScj4H7AzsApxWVb9K8krgAzRPXkHTs3Ml8M9DrVKSJKmjLmtXnUszF85g24eTnAc8jWaCwJ8AH62q61dxCUmSpDWm87IOk1XVUmDpEGqRJEkamhkPPE7y4FEWIkmSNExdnq76UZJzkrw4ySYjq0iSJGkIuoScFcAjgfcB1yT5bJL9k3S5hiRJ0hrRJaAspJkT5/Pt/jNoHiO/MslbvJ0lSZLmkxmHnKq6raq+XFUHA1sDRwLfa7f/ieZ21rnezpIkSfPBrG41VdX1VfXBqtoDeCDwFuBqYDHN7ayrh1eiJElSd3MeT1NVP62q1wD3Bd5DMyHg+tN/SpIkabTmPE9OkocAhwHPBbZqm2+a63UlSZLmYlY9OUk2TfLSJN8Dfgi8ErgP8B3gCJpxOjO91j3asTw/SHJRkte37Tu0j6wva5/kunvbvn67v6w9vmg2v0GSJPVbl8kA10nyV0lOohlz827gETQrjr8ZeEBV7VVVH6mq33ao4Q/APlX1cJq1sfZLsgfNOJ93VtWOwHXA4e35hwPXte3vbM+TJEm6ky49OVcB/wE8FSjgs8ATge2r6nVVdelsCqjG79rd9dpXAfsAJ7XtxwMHtdsHtvu0x/dNktl8tyRJ6q8uIWdL4Fzgb4H7VNVzqmpJVdVci2h7iS6gmXBwCfDfwPVVdWt7ynJgm3Z7G5qVzmmP3wBsvoprHpFkaZKlK1eunGuJkiRpLdNl4PGDquqSURRRVbcBO7fz63yB5rH0uV7zGOAYgMWLF885iEmSpLVLl56c7yT5xsgqoZl/BzgT2BPYJMlECFtIc7uM9n1bgPb4xsCvRlmXJEla+3QJOXenvU00TEkWTMyQnGQD4PHAxTRh5+ntaYcCX2y3T2n3aY+fMYxbZpIkqV+63K5aBmwxghq2Bo5Psg5N6Dqxqr6U5MfAZ5K8ETgfOLY9/1jgE0mWAb8GnjWCmiRJ0lquS8j5JPCGJDtU1c+GVUBV/RDYZRXtlwG7raL99zSLg0qSJE2py+2qdwJfA85I8swkLt0gSZLmrS49OZfSrEu1PfBpgCQrWPUSDlVV95t7eZIkSbPTJeQsGtiemHxvq1WcB81kfpIkSWPTJeTsMLIqJEmShmzGIaeqrhhlIZIkScM0q1XIJUmS5rsut6sASLIx8DyaWYkXAKdX1b+3x+5PM3bnv6pqVQOSJUmS1ohOISfJfsCngE1oBh8Xdyy3APAAmpXKn0OzSrkkSdJYzPh2VZKHAicD9wI+ADyTO56ymvBV4EbgwGEVKEmSNBtdenJeC6wP/HVVnQKQ5E69NVV1S5LzgYcPr0RJkqTuugw8fixw/kTAmcZVNOtRSZIkjU2XkLM5zSKdq3N3YIPZlSNJkjQcXULOdcDCGZx3P+Da2ZUjSZI0HF1CzrnAI5PsNNUJSR4J/DnwrbkWJkmSNBddQs77gfWAk5I8YPLBJPcFjqN5rPyDwylPkiRpdmYccqrqa8B7gYcBP07yQ5pA87gk5wA/AR4CvLOqvjmKYiVJkmaq07IOVfUy4CU0Y24eSjNPzkLgkcANwMur6h+HXaQkSVJXnZd1qKoPJTkG2Bm4L7AOcCVwblXdOuT6JEmSZqVzyAGoqj8C329fkiRJ885QViFPcs8kj0iy5TCuJ0mSNFdd1q7aO8kHkuwyqf0FNGN0zgWuSvLGIdcoSZLUWZeenBcBLwQun2hIsgPwYZoZjidWI39Nkn2HVaAkSdJsdAk5uwE/qKrrBtoOoRnX8+qq2g7Yk+ax8pcMr0RJkqTuuoScBcDySW37AL8H3gdQVUuBb+Mq5JIkacy6hJwNgVsmdpLcDVhM8+j4TQPnXYmrkEuSpDHrEnJWADsO7O9BE3wmr1O1PnATkiRJY9Ql5HwH2CXJwUnuDbyOZvzNkknnPQi4ekj1SZIkzUqXyQDfCjwFOKHdD/D9qjpr4oQkC2lCzseGVJ8kSau16KhTx13C0Fx+9JPHXUJvdFmg81zgAOBs4GKaIDP5T+KZNGtYTe7dkSRJWqM6LetQVUuYJsBU1duBt8+1KEmSpLkayrIOkiRJ882sFuhMsifwWGBh23QVcFZVfXtIdUmSJM1Jp5CTZCfgE8AjJ5ra92qPLwWeX1WXDK1CSZKkWZhxyEmyLfANYCvgN8B/csc6VotoBiU/Ejg7yW5V9fOhVipJktRBl56cN9AEnE8Af19VNwwebOfOeQ/wfOD/AIcNqUZJkqTOugw83g/4OXD45IADUFW/oVmp/Mr2XEmSpLHpEnI2Br5dVbdOdUJ77NvAvedamCRJ0lx0CTk/AzadwXkbA1fMrhxJkqTh6BJyPg48NskDpjohyQOBfWjG7cxIkm2TnJnkx0kuSvKytn2zJEuSXNq+b9q2J8l7kixL8sMku3b4DZIk6U9El5DzVuBU4KwkL24HGgOQ5F5J/hY4A/gScHSH694KvLKqHkyzsvmRSR4MHAWcXlU7Aae3+wD7Azu1ryOAD3b4LkmS9Cdiyqerklw2xaGtgPcB70tyfdu2ycDxRwCXAvebSQFVdQ1wTbv92yQXA9sAB9JMOAhwPHAW8Oq2/eNVVcB3k2ySZOv2OpIkScD0j5AvmubYxCSAqxqjsz3t5IBdJVkE7AKcA2w1EFx+QROuoAlAVw58bHnbdqeQk+QImp4etttuu9mUI0mS1mLThZwd1lgVQJJ7Ap8HXl5Vv0ly+7GqqiSdglNVHQMcA7B48eJZhS5p3BYddeq4S5CktdaUIaeq1tgTUknWowk4n6qqk9vmayduQyXZGljRtl8FbDvw8YVtmyRJ0u3Gvgp5mi6bY4GLq+odA4dOAQ5ttw8FvjjQ/vz2Kas9gBscjyNJkibrsnbVPWkGAj8CWNA2rwTOo1mB/HezrOHRwCHAj5Jc0La9luYJrROTHE4z787B7bEvA08ClgE3Ai+Y5fdKkqQeW23IaW8l/Qvwd8C9pjjtt0neD/xbVd3SpYCq+iZ3DGSebN9VnF/AkV2+Q5Ik/emZNuS0c+EsARbTBJEbgPOBa9v9LWmehtqYZh6bxyd5/KrWtpIkSVqTVteTcyLwSJolHV4F/EdV3TZ4QpJ1gL+mmSxwcfuZJw6/VEmSpJmbcuBxkgOAJ9D03OxaVZ+fHHAAquq2qjoJ2Bm4AHhc+1lJkqSxme7pqucBfwQOmcntp/acQwY+K0mSNDbThZw9ge9X1Y9nerGquojmaas951qYJEnSXEwXcrakeUy7q2XtZyVJksZmupDze2CDWVxzg/azkiRJYzNdyLkC2L3LxdrZi3cHfj6XoiRJkuZqupDzdWCrdsbhmTocuA9w2pyqkiRJmqPpQs77gFuA9yY5cHUXSnIQ8F7gZuD9wylPkiRpdqYMOVV1OfBq4B7AyUm+mOSpSRYmWa99LUzy9CT/SbOK+PrAa9rPSpIkjc20Mx5X1buTrAu8GfgrYKpJ/gLcRhNw3jncEiVJkrqb7nYVAFX1dprZjD8BXEcTaAZf17fHdqmqt42uVEmSpJlb7SrkAO2EgIcCJNkBWNAe+iXws3ZlcEmSpHljRiFnUFX9jGbBTkmSpHlrtberJEmS1kaGHEmS1EuGHEmS1EuGHEmS1EuGHEmS1EuGHEmS1EuGHEmS1EuGHEmS1EudQk6Sy5K8aVTFSJIkDUvXnpxF3LGkAwBJjkvywqFVJEmSNARThpwk/5xk3yT3Ws01DgP2GmpVkiRJczTd2lWvBwqoJBcB57Tt64y8KkmSpDmaLuQ8auC1B/Citv2wJI8CTgfOHG15kiRJszPl7aqq+m5VvaOqnl5VC4Ed2kPLgQ2BlwAn0vT2PCHJW5M8Kck9R161JEnSasx44HFVXdFufq2qtgfuTxN0AmwNvBL4T+DXSb417EIlSZK6mG7g8bFJXpjkQas6XlXLqurD7e7HgT8H/gE4FXjg0CuVJEnqYLoxOS+geXKKJDcA323bN0myblXdOnBuVdWFwIXAe5JkFMVKkiTN1HQhZweaAccTr31oxt88jeaW1Nk0g4/voqpqyHVKkiR1MmXIacfgXAF8FiDJ+sBNwI+B64EnAk+mCT4HJdkAOAs4s6p+OtqyJUmSptdl4PEf2s1vV9VewObAQTQDj9cDngF8ELg4yfJhFypJktTFrBforKrfVtUp7e7naELPU4H3A78eQm2SJEmzNt2YnE6q6gbgP9qXJEnSWHUKOVU1654fSZKkNWkYoWVv4C2z/XC7ivmKJBcOtG2WZEmSS9v3Tdv2JHlPkmVJfphk1yHUL0mSemjOIaeqzq6qS+ZwiY8B+01qOwo4vap2onlM/ai2fX9gp/Z1BM1AZ0mSpLsY++2nqvoGdx2ofCBwfLt9PM1TXBPtH6/Gd2kmJtx6zVQqSZLWJmMPOVPYqqquabd/AWzVbm8DXDlw3vK27S6SHJFkaZKlK1euHF2lkiRpXpqvIed27ezJnWdQrqpjqmpxVS1esGDBCCqTJEnz2XwNOddO3IZq31e07VcB2w6ct7BtkyRJupP5GnJOAQ5ttw8FvjjQ/vz2Kas9gBsGbmtJkiTdbmiTAc5WkhOAxwJbtMtB/CtwNHBiksNp1s86uD39y8CTgGXAjTQrpUuSJN3F2ENOVT17ikP7ruLcAo4cbUWSJKkP5uvtKkmSpDkx5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4y5EiSpF4a+wKd0rAtOurUcZcgSZoH7MmRJEm9ZMiRJEm95O0qSZLmkb7ccr/86CePuwR7ciRJUj8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi8ZciRJUi+tO+4CNH8sOurUcZcgSdLQ2JMjSZJ6yZAjSZJ6yZAjSZJ6yZAjSZJ6yZAjSZJ6aa0MOUn2S3JJkmVJjhp3PZIkaf5Z60JOknWA9wP7Aw8Gnp3kweOtSpIkzTdrXcgBdgOWVdVlVXUz8BngwDHXJEmS5pm1cTLAbYArB/aXA7uPqRYn0JMkaZ5aG0POjCQ5Ajii3f1dkktG9FVbAL8c0bXnC39jP/gb+8Hf2A+9/415y0h/4/YzOWltDDlXAdsO7C9s2+6kqo4Bjhl1MUmWVtXiUX/POPkb+8Hf2A/+xn7wN64Za+OYnO8BOyXZIcndgWcBp4y5JkmSNM+sdT05VXVrkpcCXwPWAY6rqovGXJYkSZpn1rqQA1BVXwa+PO46WiO/JTYP+Bv7wd/YD/7GfvA3rgGpqnHXIEmSNHRr45gcSZKk1TLkzEKSbZOcmeTHSS5K8rJx1zQKSe6R5NwkP2h/5+vHXdMoJFknyflJvjTuWkYlyeVJfpTkgiRLx13PKCTZJMlJSX6S5OIke467pmFK8oD2z2/i9ZskLx93XcOW5B/av28uTHJCknuMu6ZhS/Ky9vdd1Jc/wyTHJVmR5MKBts2SLElyafu+6Zquy5AzO7cCr6yqBwN7AEf2dGmJPwD7VNXDgZ2B/ZLsMeaaRuFlwMXjLmIN2Luqdh73I50j9G7gq1X1QODh9OzPtKouaf/8dgYeAdwIfGHMZQ1Vkm2AvwcWV9VDaR4uedZ4qxquJA8F/oZm9v6HAwck2XG8VQ3Fx4D9JrUdBZxeVTsBp7f7a5QhZxaq6pqq+n67/Vuav0y3GW9Vw1eN37W767WvXg3iSrIQeDLwkXHXotlLsjHwGOBYgKq6uaquH29VI7Uv8N9VdcW4CxmBdYENkqwLbAhcPeZ6hu1BwDlVdWNV3QqcDTx1zDXNWVV9A/j1pOYDgePb7eOBg9ZoURhy5izJImAX4JzxVjIa7a2cC4AVwJKq6tvvfBfwKuCP4y5kxAo4Lcl57WzgfbMDsBL4aHvr8SNJNhp3USP0LOCEcRcxbFV1FfA24OfANcANVXXaeKsauguBv0iyeZINgSdx5wlu+2Srqrqm3f4FsNWaLsCQMwdJ7gl8Hnh5Vf1m3PWMQlXd1naPLwR2a7taeyHJAcCKqjpv3LWsAXtV1a7A/jS3Vx8z7oKGbF1gV+CDVbUL8D+MoWt8TWgnQX0K8Llx1zJs7ZiNA2lC658BGyV53nirGq6quhh4C3Aa8FXgAuC2sRa1BlTzKPcavxNgyJmlJOvRBJxPVdXJ465n1Nqu/zO56z3XtdmjgackuZxmNft9knxyvCWNRvsvZKpqBc04jt3GW9HQLQeWD/Q0nkQTevpof+D7VXXtuAsZgccBP6uqlVV1C3Ay8Kgx1zR0VXVsVT2iqh4DXAf8dNw1jci1SbYGaN9XrOkCDDmzkCQ09/4vrqp3jLueUUmyIMkm7fYGwOOBn4y3quGpqtdU1cKqWkTT/X9GVfXqX40ASTZKcq+JbeAJNF3mvVFVvwCuTPKAtmlf4MdjLGmUnk0Pb1W1fg7skWTD9u/ZfenZAHKAJFu279vRjMf59HgrGplTgEPb7UOBL67pAtbKGY/ngUcDhwA/aserALy2nYm5T7YGjk+yDk0gPrGqevuYdY9tBXyh+X8G6wKfrqqvjrekkfg74FPt7ZzLgBeMuZ6ha0Pq44H/Ne5aRqGqzklyEvB9mqdYz2cezJo7Ap9PsjlwC3BkHwbJJzkBeCywRZLlwL8CRwMnJjkcuAI4eI3X5YzHkiSpj7xdJUmSesmQI0mSesmQI0mSesmQI0mSesmQI0mSesmQI6mTdkXzmvT6fZKfJzkxyV+Ou8b5JMkn2/9GvZuDSZrvnCdH0mx9jWY9GoBNadZwewbwjCSvqKp3jq0yScKQI2n2jq6qsyZ22qVO3g28GDg6yeeqavm4ipMkb1dJGop2raFXAr8F7k6zfIQkjY0hR9LQVNVN3LHY4FaTj6fxnCRLkvwqyc1Jrkjy4XYdn8Fzd2vHsnxrFdd5T3vs5napg8FjT2mPnTyp/RlJPprkx0mub8cRXZrkfUm2WdXvSfLN9lp7Jdk7yVeS/DLJH9tV7CfOu2eSNye5LMkfklzZXnezmf/XkzRshhxJw7Zx+36nVbLb21lfAD5Fs7L0RTQL9t0EHAGcn2SXgY+cB9wA7DaxwOiAfdv39YDHTHHs65PaTwSeTtPTtKQ9vgFwZPvd95vmNz0bOB3Ypv3sGTTrDtHWdjZwFLA58BXge8BzgO9yx38PSWuYIUfS0CR5CLADcDNw2qTDbwYOBM4Edqqqx1TVM6rqgcA/AJsBn2kXhKWqbgPOohk7ePsTW0m2Bh4M/Khtetyk75kIOadPan8WsGVV7d5+7wHAIppFBBcA75rmp70EeFFV/XlVPbuqHldVX2uPvRHYFbgA2LGqDqqqpwL3A34FHLDqS0oaNUOOpDlLsmmS/YGTaf5eefngoOMkC4CX0vTMHFxVVw9+vqreRfO01v2581ieid6YwSCzT/v+bmDF4LEk9wEeAiyvqksmfcdn29tpg223Aq+l6XUYq/naAAADZ0lEQVTaP8mGU/zEr1TVcav43RsBL2p3/66qVg5c+zqacCRpTHy6StJsnZlkctsfgP0Hejkm7AOsD3y1qn45xfXOBp4I7Elzywfu6I3Zd+C8ie0lNAHnmUm2rKoV3BGAJvfiAJDkAe137Ajckzv+oXc3YB2a3pcfreKjJ6+iDeCRwIbAFVX1zckHq+r8JBfRBC9Ja5ghR9JsTcyTE+A+NGNj7gF8PMmjq2rZwLn3bd8PTFKrue6CiY2qujjJ1cBDk2xVVdfShJxLq+rnSb5OcxtqX+AEprhV1Y4H+iBw+Gq++95TtF8xRfvC9v1n01zzcgw50lgYciTN1uR5cramCT4PAz6VZI+qmgg067TvPwHOWc11z520fzpwCLBvknOB7YAPDRyDpkdnypADvIIm4Cynecz9O8CKqvpDW/u5NL0yd+maat00RbukecyQI2koquqaJAcDPwR2A54LfLI9fGX7fkFVHdbx0hMh53HAxFNWX2+/8/Ik/00TgO4HbA9cPHnMD81MzAB/U1VfXcV37NixpglXte+LpjlnumOSRsiBx5KGpqp+Anyg3f23JBP/kFoC3Ao8IclUt4SmMjguZ1/gjzRPaA0e3x7420nnD5qYr+bKyQfaAdObdqxpwvdoenkWJXnUKq69M96qksbGkCNp2N5EMxfN/Wh6YGh7Vj5EEzZOSXL/yR9KslGS57VPYt2ufUrrEprbVH8FnF9Vvx44ZeIJrJe276sKOT9p31+cgdHSSXbijlDWWVX9Dph46uq9SbYYuPYmwPtne21Jc2fIkTRU7WPUb2t3//dAb84rgc/TzHlzUZLvtauWn9iOifk18AlWPXneRHC5B3cNMWcA1R67jTv38kz4vzST9x0JXJzkM0mWABcCl7H6cULTeS3NHDm7AsuSnJzk8+11FwBfmsO1Jc2BIUfSKLyDZu6Z+wKHAlTVzVX1dOAg4Ms0TyYdRPPY9wY0MyEfRPM00mSDweZOMxlX1a9oQgbAeVV1w+QPt493795+7ybAU4A/A94A7E9zK21Wquo3NE+W/TtwPc3kf7vTzLC8J83cQJLGIHc8/CBJktQf9uRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqReMuRIkqRe+v/BZVqIt0hHaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,5))\n",
    "plt.xlabel(\"Reward\",fontsize=22)\n",
    "plt.ylabel(\"# Observations\",fontsize=22)\n",
    "plt.hist(reward_test,bins=9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10 actions x 2 columns\n",
    "# Columns: Count #, Avg Reward\n",
    "record = np.zeros((n,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_arm(record):\n",
    "    arm_index = np.argmax(record[:,1],axis=0)\n",
    "    return arm_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_record(record,action,r):\n",
    "    new_r = (record[action,0] * record[action,1] + r) / (record[action,0] + 1)\n",
    "    record[action,0] += 1\n",
    "    record[action,1] = new_r\n",
    "    return record"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x12455c4e0>"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAFACAYAAABuonzYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHbJJREFUeJzt3XuUZWV55/Hv00UBp9GhuJSXLkC8kGaJIG1qlEQmS2C0MYD2oFFQx2ic6Zk1mRlwMu3YWWYRXRpMOvEyScY1DSEaxbtNiU5iwygJxiWM1VRrcatwEZVqlBIsFagFRfnMH2dXp7qty6mqs88+p873s1atPmfXPud9eJs+9at3v/t9IzORJEmqyrqqC5AkSd3NMCJJkiplGJEkSZUyjEiSpEoZRiRJUqUMI5IkqVKGEUmSVCnDiCRJqpRhRJIkVeqQqguY69hjj80TTzyx6jIkSVIT7Nmz58eZ2b/UeW0VRk488USGh4erLkOSJDVBRHyvkfNKvUwTEW+PiNsi4taI+FREHF5me5IkqfOUFkYiYgD4r8BgZr4A6AEuKqs9SZLUmcqewHoIUIuIQ4D1wL6S25MkSR2mtDCSmePAnwLfBx4AfpqZ1x18XkRsjYjhiBiemJgoqxxJktSmyrxMcxTwauDZwAbgiIh408HnZebOzBzMzMH+/iUn3EqSpDWmzMs0/xr4bmZOZOY0sAv49RLbkyRJHajMMPJ94IyIWB8RAZwD3FFie5IkqQOVOWfkZuDzwC3AaNHWzrLakyRJnanURc8y8zLgsjLbkJphaGScHbvH2Dc5xYa+Gts2b2TLpoEVnydJjRoaGecPr72Nyanplre9LuAXCQMVf55FZlbS8HwGBwfTFVjVSkMj42zf9R2mpn9xwPEA3njGCbx3y6mLflAEkFT/D1lSe6gyWKxWrbeHyy88tamfYxGxJzMHlzzPMKKFNGMUYDn/MI84tIf3/Zvm/kNYbU3LddT6Xi674BSAA9qYPW5YkTpLJ4eLlRjoq/GNd57dtPczjGhFFvuHt5zUvNCIw2rM9wN9qQ+Ko9b3ct5pz+TL336grT5MGg0ns4FwfHJq/yhMo++1kjC50g/e+fp5fW99Stpjc/4fWGkoKysYlx0SV9Kf8/VbGdYFvOEl9dG/srXqB/pCvwwA9K6Dkrt0TQjgu+8/r3nvZxhRI941NMrVN31/3h9y81koNc/9oanG9PYEO177QuCXPzjVnuYbvSsjeFelVUFI7cuREQwjrfbGK77JN+55eNmv+9DrT9//W+qRtV4efXza3zgkqcMF8MHXn17JnJFS76ZRe1rtb3KXfmbv/sf+Ni9JnW920n5V89oMI13mXUOjfOKm71ddhiTpIMudS7baOVTttEyBl2m6gPM5ljY7mW/wWUd35IRYSe3FO+jqvEzTQZpx18PchWvOOrm/7X5YHnbIOv74NafNe8fH9l2jTE3PtLymhT4sGvnwmL0DYam7XQ7tCQ5ZF04IlDqc4aJcjoxUbKFJpIutudGKSy2zIwCrbWfu4mELmfsDvSeCmcz9i4gBDYWV9b3rOKy3h588Nr3/PQ4OB1V/mAyNjLPtc99m+hcr/zfXE8HFLzl+ycXYVms5w8VL1dBuo0lV3DHSSH8e3Je962Am679kzP17X60q182Y2w9zfwk7stZLBEw+Nr3sSwaL/ffM9/9e1Z8D3ca7aTrAUqFi9tbPuf9oygwib5onNGx6z3X85LHGP7TKWJG03a5trsZSH5x+SK7cwX3bzP5sNHR16t9fmX2n7mYYaTPz/UD9b5/dy1K/JPdE8ItMNvTVWH/oOu568NGm17bYKEyjv8374SVJOphzRtrIwfMixienDrg9djEzRVgsa/LpfKMhc82Gi1avXClJ6h6GkZINjYzz9s/upY0GoIDG5nLM2rJpwNAhSSqNYaREs5c42iWItMtW0ZIkzWUYKdG7v3Tbqu6caNQRh/bw6BMzC26kVsa20JIkNYthpCRDI+PLugtlpV763KO5+t//2i+1vVbuPpEkrX2GkZK8+0u3ld5GrXfdLwURcI6HJKmzrKu6gLVoJaMitd4ejlrf2/D5AVx+4WnLrEySpPZjGGmyoZFx3r7EbbuzoaMnAqhPKL38wlO57IJTqPX2NNROlbsrSpLUTF6maaJGVkf90OtPXzJEzJ3vcfA+M67vIUlaa1yBtQnqi5p9h6kG9rq47/3ntaAiSZKq5wqsLbKczc/6ao3PCZEkqVuUNmckIjZGxN45Xz+LiEvLaq8qy1lL5A9fdUrJ1UiS1HlKGxnJzDHgdICI6AHGgWvKaq8K7xoabfiumTc54VSSpHm16jLNOcA9mfm9FrVXuqGRca5eYrIqLL4jriRJal0YuQj41HzfiIitwFaAE044oUXlrN6O3WPzLr0+11I74kqSpBbcTRMRhwL7gFMy80eLndtud9McvKz6WSf3c8OdE+ybnFo0iETAB1+39C28kiStZe10N80rgVuWCiLtpn677ihT0zMAjE9OLbmGCNRXRjWISJLUuFaswHoxC1yiaWc7do/tDyLL4cqokiQtT6lhJCKOAF4O7CqznTLsm5xa0eucIyJJ0vKUGkYy89HMPCYzf1pmO2XoW8amdbMG+molVCJJ0trmCqxzzE5YHV/hqMi2zRubXJEkSWufYaRw8ITV5eqr9TpXRJKkFWjFBNaOsNIJq1C/g8al3iVJWhnDSGGlE1YD76CRJGk1uv4yzew8kZUs/TbQV2Pb5o0GEUmSVqGrw8hK54n01XrZe9krSqpKkqTu0tWXaVYyT6TW2+P8EEmSmqirR0aWM08kgA1elpEkqem6Ooxs6Ks1tKbIQF+Nb7zz7BZUJElS9+nqyzSNLFJW6+1xMTNJkkrU1WFkqcstPRFcfuGpXpaRJKlEXR1GYOH9ZAL4s9e90CAiSVLJujqMDI2M8+jjT/7ScRcykySpdbp2AutCa4wctb6Xyy44xSAiSVKLdO3IyEJrjKw/9BCDiCRJLdS1YWShW3pXukeNJElama4MI0Mj48QC39uwwIRWSZJUjq4MIwttjBc0tvaIJElqnq4MIwtdikmWXntEkiQ1V1eGkYUuxSy05ogkSSpPV4aRbZs3UuvtOeCYy75LklSNrlxnZPZSzI7dY+ybnHI3XkmSKlRqGImIPuBK4AXUp2T8TmZ+s8w2G7Vl04DhQ5KkNlD2yMiHga9k5msj4lBgfcntSZKkDlNaGImII4HfAN4CkJlPAE+U1Z4kSepMZU5gfTYwAfx1RIxExJURcUSJ7UmSpA5UZhg5BHgR8JHM3AQ8Crzz4JMiYmtEDEfE8MTERInlSJKkdlTmnJH7gfsz8+bi+eeZJ4xk5k5gJ8Dg4OB8C6M23dDIuHfSSJLUJkobGcnMHwI/iIjZxTvOAW4vq71GDY2Ms33XKOOTUyT1DfO27xplaGS86tIkSepKZS969l+AqyPiO8DpwB+V3N6SduweY2p65oBjU9Mz7Ng9VlFFkiR1t1Jv7c3MvcBgmW0s10L70ix0XJIklavrloNfaF+ahY5LkqRydV0YcV8aSZLaS9ftTeO+NJIktZeuCyPgvjSSJLWTrrtMI0mS2othRJIkVcowIkmSKmUYkSRJleq6CazuSyNJUnvpqjAyuy/N7HLws/vSAAYSSZIq0lWXadyXRpKk9tNVYcR9aSRJaj9dFUbcl0aSpPbTVWHEfWkkSWo/XTWB1X1pJElqP10VRsB9aSRJajdddZlGkiS1H8OIJEmqlGFEkiRVyjAiSZIqZRiRJEmVMoxIkqRKGUYkSVKlSl1nJCLuA34OzABPZuZgme1JkqTO04pFz87KzB+3oB1JktSBvEwjSZIqVXYYSeC6iNgTEVtLbkuSJHWgsi/TnJmZ4xHxNOD6iLgzM2+ce0IRUrYCnHDCCSWXI0mS2k2pIyOZOV78+SBwDfDiec7ZmZmDmTnY399fZjmSJKkNlRZGIuKIiHjq7GPgFcCtZbUnSZI6U5mXaZ4OXBMRs+18MjO/UmJ7SxoaGWfH7jH2TU6xoa/Gts0b2bJpoMqSJEnqeqWFkcy8F3hhWe+/XEMj42zfNcrU9AwA45NTbN81CmAgkSSpQl1za++O3WP7g8isqekZduweq6giSZIEXRRG9k1OLeu4JElqjQUv00TE0Yu9MDMfbn455dnQV2N8nuCxoa9WQTWSJGnWYiMje4Dh4s8J4J+Au4rHe8ovrbm2bd5IrbfngGO13h62bd5YUUWSJAkWCSOZ+ezMfA7wf4ELMvPYzDwGOB+4rlUFNsuWTQNcfuGpDPTVCGCgr8blF57q5FVJkioWmbn4CRGjmXnqUseaYXBwMIeHh5v9tpIkqQIRsSczB5c6r5Fbe/dFxLuATxTP3wjsW01xkiRJsxq5m+ZioJ/6cu67iscXl1mUJEnqHouOjERED/D7mXlJi+qRJEldZtGRkcycAc5sUS2SJKkLNTJnZCQirgU+Bzw6ezAzd5VWlSRJ6hqNhJHDgYeAs+ccS+rzRyRJklZlyTCSmW9tRSGSJKk7LRlGIuJw4G3AKdRHSQDIzN8psS5JktQlGrm19+PAM4DNwD8AxwE/L7MoSZLUPRoJI8/LzD8AHs3MjwHnAS8ptyxJktQtGgkj08WfkxHxAuBI4GnllSRJkrpJI3fT7IyIo4A/AK4FnlI8liRJWrVG7qa5snj4D8Bzyi1HkiR1m0buprkHuAn4OvD1zLyt9KokSVLXaGTOyPOB/w0cA+yIiHsi4ppyy5IkSd2ikTAyQ30S6wzwC+DB4kuSJGnVGpnA+jNgFPgAcEVmPlRuSZIkqZs0MjJyMXAj8J+AT0fEuyPinEYbiIieiBiJiC+vtEhJkrR2NXI3zReBL0bEycArgUuBdwC1Btu4BLgD+BcrLVKSJK1dS46MRMQXIuJu4MPAeuDNwFGNvHlEHEd9xdYrlzpXkiR1p0bmjFwOjGTmzAre/0PUR1GeutAJEbEV2ApwwgknrKAJSZLUyRqZM3I7sD0idgJExEkRcf5SLyrOeTAz9yx2XmbuzMzBzBzs7+9vqGhJkrR2NBJG/hp4Avj14vk48N4GXvdS4FURcR/waeDsiPjESoqUJElrVyNh5LmZ+ScUG+Zl5mNALPWizNyemcdl5onARcDXMvNNqylWkiStPY2EkSciogYkQEQ8F3i81KokSVLXaGQC62XAV4DjI+Jq6pdf3rKcRjLz74G/X2ZtkiSpCzSyzsj1EXELcAb1yzOXZOaPS69MkiR1hUYu05CZD2Xm/8nMLwNHR8QVJdclSZK6xIJhJCJOi4jrIuLWiHhvRDwzIr4AfI367b6SJEmrttjIyBXAJ4HXABPAXuAe4HmZ+cEW1CZJkrrAYnNGDsvMjxaPxyLiksx8RwtqkiRJXWSxMHJ4RGzin9cUeXzu88y8peziJEnS2rdYGHkA+MCc5z+c8zyBs8sqSpIkdY8Fw0hmntXKQso2NDLOjt1j7JucYkNfjW2bN7Jl00DVZUmS1PUaWfSs4w2NjLN91yhT0/WNh8cnp9i+axTAQCJJUsUaWmek0+3YPbY/iMyamp5hx+6xiiqSJEmzuiKM7JucWtZxSZLUOktepomIF81z+KfA9zLzyeaX1Hwb+mqMzxM8NvTVKqhGkiTN1cjIyP8CbgJ2Ul8I7ZvA56ivPfKKEmtrmm2bN1Lr7TngWK23h22bN1ZUkSRJmtVIGNkHbMrMwcz8VWATcC/wcuBPyiyuWbZsGuDyC09loK9GAAN9NS6/8FQnr0qS1AYauZvmVzLzttknmXl7RJycmfdGxGKvaytbNg0YPiRJakONhJHbIuIjwKeL568Hbo+Iw4Dp0iqTJEldoZHLNG8B7gYuLb7uLY5NA2tqYTRJktR6jYyMvBL4i8z8s3m+90iT65EkSV2mkZGRC4B/ioiPR8T5EdEVq7ZKkqTWWDKMZOZbgedRv533YuCeiLiy7MIkSVJ3aGiUIzOnI+LvqO/WWwO2AP+uzMIkSVJ3WHJkJCJeGREfBe4CXgNcCTyj5LokSVKXaGRk5M3AZ4D/kJmPl1yPJEnqMkuGkcy8eO7ziDgTuDgzf3ex10XE4cCNwGFFO5/PzMtWUeuKDY2Ms2P3GPsmp9jQV2Pb5o0ugCZJUptoaM5IRGwC3gD8FvBdYFcDL3scODszH4mIXuAfI+LvMvOmFVe7AkMj42zfNcrU9AwA45NTbN81CmAgkSSpDSw4ZyQifiUiLouIO4E/B74PRGaelZl/vtQbZ93sOiS9xVc2o+jl2LF7bH8QmTU1PcOO3WOtLkWSJM1jsQmsdwJnA+dn5plFAJlZ5PxfEhE9EbEXeBC4PjNvnuecrRExHBHDExMTy3n7huybnFrWcUmS1FqLhZELgQeAGyLiiog4B1jWzniZOZOZpwPHAS+OiBfMc87OYkfgwf7+/uW8fUM29NWWdVySJLXWgmEkM4cy8yLgZOAG6vvSPC0iPhIRr1hOI5k5WbzHuaspdiW2bd5IrbfngGO13h62bd7Y6lIkSdI8GlmB9dHM/GRmXkB9hGME+B9LvS4i+iOir3hcA15O/dJPS23ZNMDlF57KQF+NAAb6alx+4alOXpUkqU1EZjlzSiPiNOBjQA/10PPZzHzPYq8ZHBzM4eHhUuqRJEmtFRF7MnNwqfNK2/QuM78DbCrr/SVJ0trQyK69kiRJpTGMSJKkShlGJElSpQwjkiSpUoYRSZJUKcOIJEmqlGFEkiRVqrR1RtrF0Mg4O3aPsW9yig19NbZt3ujqq5IktZE1HUaGRsbZvmuUqen6ZsPjk1Ns3zUKYCCRJKlNrOnLNDt2j+0PIrOmpmfYsXusoookSdLB1nQY2Tc5tazjkiSp9dZ0GNnQV1vWcUmS1HprOoxs27yRWm/PAcdqvT1s27yxoookSdLB1vQE1tlJqt5NI0lS+1rTYQTqgcTwIUlS+1rTl2kkSVL7M4xIkqRKGUYkSVKlDCOSJKlShhFJklQpw4gkSaqUYUSSJFWqtDASEcdHxA0RcXtE3BYRl5TVliRJ6lxlLnr2JPB7mXlLRDwV2BMR12fm7SW2KUmSOkxpIyOZ+UBm3lI8/jlwB+BSqJIk6QAtmTMSEScCm4CbW9GeJEnqHKWHkYh4CvAF4NLM/Nk8398aEcMRMTwxMVF2OZIkqc2UGkYiopd6ELk6M3fNd05m7szMwcwc7O/vL7McSZLUhsq8myaAvwLuyMwPlNWOJEnqbGWOjLwU+LfA2RGxt/j6zRLbkyRJHai0W3sz8x+BKOv9JUnS2uAKrJIkqVKGEUmSVCnDiCRJqpRhRJIkVcowIkmSKmUYkSRJlTKMSJKkShlGJElSpQwjkiSpUoYRSZJUKcOIJEmqlGFEkiRVyjAiSZIqZRiRJEmVMoxIkqRKGUYkSVKlDCOSJKlShhFJklQpw4gkSaqUYUSSJFXKMCJJkiplGJEkSZUyjEiSpEqVFkYi4qqIeDAibi2rDUmS1PnKHBn5KHBuie8vSZLWgNLCSGbeCDxc1vtLkqS1ofI5IxGxNSKGI2J4YmKi6nIkSVKLVR5GMnNnZg5m5mB/f3/V5UiSpBarPIxIkqTuZhiRJEmVKvPW3k8B3wQ2RsT9EfG2stqSJEmd65Cy3jgzLy7rvSVJ0trhZRpJklQpw4gkSaqUYUSSJFXKMCJJkiplGJEkSZUyjEiSpEoZRiRJUqUMI5IkqVKGEUmSVCnDiCRJqpRhRJIkVcowIkmSKmUYkSRJlTKMSJKkShlGJElSpQwjkiSpUoYRSZJUKcOIJEmqlGFEkiRVyjAiSZIqZRiRJEmVMoxIkqRKGUYkSVKlDinzzSPiXODDQA9wZWa+v8z2DjY0Ms6O3WPsm5xiQ1+NbZs3smXTQCtLkCRJSygtjERED/CXwMuB+4FvRcS1mXl7WW3ONTQyzvZdo0xNzwAwPjnF9l2jAAYSSZLaSJmXaV4M3J2Z92bmE8CngVeX2N4Bduwe2x9EZk1Nz7Bj91irSpAkSQ0oM4wMAD+Y8/z+4tgBImJrRAxHxPDExETTGt83ObWs45IkqRqVT2DNzJ2ZOZiZg/39/U173w19tWUdlyRJ1SgzjIwDx895flxxrCW2bd5IrbfngGO13h62bd7YqhIkSVIDyryb5lvASRHxbOoh5CLgDSW2d4DZSareTSNJUnsrLYxk5pMR8Z+B3dRv7b0qM28rq735bNk0YPiQJKnNlbrOSGb+LfC3ZbYhSZI6W+UTWCVJUnczjEiSpEoZRiRJUqUMI5IkqVKGEUmSVCnDiCRJqpRhRJIkVSoys+oa9ouICeB7Jbz1scCPS3hfzc/+bh37unXs69ayv1unzL5+VmYuufFcW4WRskTEcGYOVl1Ht7C/W8e+bh37urXs79Zph772Mo0kSaqUYUSSJFWqW8LIzqoL6DL2d+vY161jX7eW/d06lfd1V8wZkSRJ7atbRkYkSVKbMoxIkqRKrfkwEhHnRsRYRNwdEe+sup5OFxFXRcSDEXHrnGNHR8T1EXFX8edRxfGIiP9Z9P13IuJF1VXeeSLi+Ii4ISJuj4jbIuKS4rj9XYKIODwi/l9EfLvo73cXx58dETcX/fqZiDi0OH5Y8fzu4vsnVll/J4qInogYiYgvF8/t6xJExH0RMRoReyNiuDjWVp8jazqMREQP8JfAK4HnAxdHxPOrrarjfRQ496Bj7wS+mpknAV8tnkO9308qvrYCH2lRjWvFk8DvZebzgTOA3y3+/7W/y/E4cHZmvhA4HTg3Is4A/hj4YGY+D/gJ8Lbi/LcBPymOf7A4T8tzCXDHnOf2dXnOyszT56wn0lafI2s6jAAvBu7OzHsz8wng08CrK66po2XmjcDDBx1+NfCx4vHHgC1zjv9N1t0E9EXEM1tTaefLzAcy85bi8c+pf2gPYH+Xoui3R4qnvcVXAmcDny+OH9zfs38PnwfOiYhoUbkdLyKOA84DriyeB/Z1K7XV58haDyMDwA/mPL+/OKbmenpmPlA8/iHw9OKx/d8kxbD0JuBm7O/SFJcN9gIPAtcD9wCTmflkccrcPt3f38X3fwoc09qKO9qHgHcAvyieH4N9XZYErouIPRGxtTjWVp8jh5TdgLpLZmZEeL94E0XEU4AvAJdm5s/m/kJofzdXZs4Ap0dEH3ANcHLFJa1JEXE+8GBm7omIl1VdTxc4MzPHI+JpwPURcefcb7bD58haHxkZB46f8/y44pia60ezw3jFnw8Wx+3/VYqIXupB5OrM3FUctr9LlpmTwA3Ar1Efpp79xW1un+7v7+L7RwIPtbjUTvVS4FURcR/1y+dnAx/Gvi5FZo4Xfz5IPWS/mDb7HFnrYeRbwEnFDO1DgYuAayuuaS26Fvjt4vFvA1+cc/zNxezsM4CfzhkW1BKKa+J/BdyRmR+Y8y37uwQR0V+MiBARNeDl1Ofp3AC8tjjt4P6e/Xt4LfC1dBXJhmTm9sw8LjNPpP65/LXMfCP2ddNFxBER8dTZx8ArgFtps8+RNb8Ca0T8JvVrkz3AVZn5vopL6mgR8SngZdS3nP4RcBkwBHwWOAH4HvC6zHy4+GH6F9TvvnkMeGtmDldRdyeKiDOBrwOj/PN19d+nPm/E/m6yiDiN+kS+Huq/qH02M98TEc+h/tv70cAI8KbMfDwiDgc+Tn0uz8PARZl5bzXVd67iMs1/z8zz7evmK/r0muLpIcAnM/N9EXEMbfQ5subDiCRJam9r/TKNJElqc4YRSZJUKcOIJEmqlGFEkiRVyjAiSZIqZRiRtGoRMVPsCHprRHwuItYXxx9Z6rWSZBiR1AxTxY6gLwCeAP5j1QVJ6hyGEUnN9nXgeXMPRMRTIuKrEXFLRIxGxKuL4++JiEvnnPe+iLgkIp4ZETfOGW35Vy3+b5DUQi56JmnVIuKRzHxKsW/IF4CvZOZHDjq+vtjo71jgJuAk4FnArsx8UUSsA+6ivm/GW4DDi5Uie4rX/ryS/zhJpXPXXknNUIuIvcXjr1PfU2euAP4oIn6D+tL2A9S3ML8vIh6KiE3UtzAfycyHIuJbwFXFRoFDmbkXSWuWYURSM0xl5umLfP+NQD/wq5k5XezWenjxvSupj4Q8A7gKIDNvLILLecBHI+IDmfk3ZRUvqVrOGZHUCkcCDxZB5Czql2dmXUN9U65/CewGiIhnAT/KzCuoh5UXtbheSS3kyIikVrga+FJEjALDwJ2z38jMJyLiBmAyM2eKwy8DtkXENPAI8OYW1yuphZzAKqlSxcTVW4Dfysy7qq5HUut5mUZSZSLi+cDdwFcNIlL3cmREkiRVypERSZJUKcOIJEmqlGFEkiRVyjAiSZIqZRiRJEmV+v8gbxomeufH2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(1,1)\n",
    "ax.set_xlabel(\"Plays\")\n",
    "ax.set_ylabel(\"Avg Reward\")\n",
    "fig.set_size_inches(9,5)\n",
    "rewards = [0]\n",
    "for i in range(500):\n",
    "    if random.random() > 0.2:\n",
    "        choice = get_best_arm(record)\n",
    "    else:\n",
    "        choice = np.random.randint(10)\n",
    "    r = get_reward(probs[choice])\n",
    "    record = update_record(record,choice,r)\n",
    "    mean_reward = ((i+1) * rewards[-1] + r)/(i+2)\n",
    "    rewards.append(mean_reward)\n",
    "ax.scatter(np.arange(len(rewards)),rewards)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(av, tau=1.12):\n",
    "    softm = ( np.exp(av / tau) / np.sum( np.exp(av / tau) ) )\n",
    "    return softm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = np.random.rand(n)\n",
    "record = np.zeros((n,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x123fbcb38>"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAFACAYAAABuonzYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGYBJREFUeJzt3X+U5XV93/Hnm9kBZtEwK4zRHdDFQJaDGFmyNVRtjkDNqoBsDW0gWiMxh9M2bSEnGevmmIOmWtNMi7FJyumKRGOwNsoyojUOVEmkP6DuMujIj6lAFJlFGYFBgYkMw7t/3O+ss+PuzJ3d+72fO3Ofj3P27P1+7p37ffNZzp3X/fz4fiMzkSRJKuWI0gVIkqTuZhiRJElFGUYkSVJRhhFJklSUYUSSJBVlGJEkSUUZRiRJUlGGEUmSVJRhRJIkFbWudAELHX/88blp06bSZUiSpBbYs2fP9zNzYLnXdVQY2bRpE7t37y5dhiRJaoGI+HYzr3OaRpIkFWUYkSRJRRlGJElSUYYRSZJUlGFEkiQVZRiRJElFGUYkSVJRhhFJklRUR130TJK0urxnZJzrbnuQrI6PObKHD/yjV7B9yyAAI2OTDI9OsHd6ho39fQxt27zvuXaYP//k9EzbzrkabVjfy5UXvLyt/zYLRWYu/6o22bp1a3oFVkla2sjYJO+98S6mZ2ZLl6I1pLcnGL7olS0NJBGxJzO3Lvc6R0a61MG+LSz+ViO10+Jv0ZuO6+N/3/8Yh/KV6YiA5xIGD/Hb+MjYJDt2fZ2Z2ecO4ezS6jM7lwyPThT5/HdkpIvU8W3qcIf2lqtpufdvZgi4VcPEB3ofoKk+Xd97BEf19jD99GzTNfjtV1K7BfC3f3Be696vyZERw8gq0swvpw3rezntxc8/5G+Th+Ng30QXjsIEtL0uSVJzBvv7+F/vPqdl72cYWQNceCVJahfXjGgfh+YlSe1WejeNYaQww4ektWB9b+OyVU936ILfw13Q3Izl1qeV3ubcyZymKcSV+pIOV+lvs9JynKbpUJ0WQua/LfT39fLMs3Md+61G3e1Av3QXf8s8+9QBbrl3isnpGXoimMtsyYJpf+FL9TOM1GTxVQmP7Akyk074Xd/sh+vCD/tj+3qJYN/W1LNPHeDzX3u4ZdNLRwT86i+8hPdvf8W+c6+m6av5PgV+4hfk4faTvwwPbPuWQftEWiOcpmmxdo18LJ7/hJ+83kW7fok1u3X3cOZs655rdS5XklrPrb0FNILIODOzcy1933YsvJIkqdVcM1LA+z53V8uCiEPzkqRuYRhpkfeMjPP404e3vsEAIknqRoaRFVq8sHJ97xHMzj13yAtTvTGdJKnbGUaW0MyOjkPdCmsIkSSpwTByECNjkwx9+mvMPtfaBb5OxUiStD/DyEEMj060JIgYPiRJWpph5CBacafct53144t4SZKkAzuidAGdaGRskjjM9zCISJLUHEdGDmB4dOKQ72fhwlRJklam1jASEb8F/AaNK4SPA5dm5t/Vec5WOJQpmgDe6miIJEkrVts0TUQMAv8a2JqZpwM9wMV1na9Vlpqi6e/rZbC/j6geb1jfS9C4TPuHfuUMg4gkSYeg7mmadUBfRMwC64G9NZ/vsL3vc3cdcIomgPe+2V0xkiS1Wm0jI5k5CfwH4EHgYeCJzLxp8esi4rKI2B0Ru6empuoqpykjY5MHvaR7gkFEkqQa1DlNswG4EDgJ2AgcExFvW/y6zNyZmVszc+vAwEBd5TRleHTioM8N9ve1sRJJkrpHnVt7/yHwt5k5lZmzwC7g1TWe77DtXWLh6tC2zW2sRJKk7lHnmpEHgbMiYj0wA5wL7K7xfCs2MjbJ8OgEe6dn2Njfx/oje3jqmbmfeF1/X69TNJIk1aS2MJKZt0fEZ4A7gGeBMWBnXedbqfeMjHPdbQ/uW6x6sO28vT3Be9/88vYVJklSl6l1N01mXglcWec5DsXI2OR+QWQpxxy5zlERSZJq1JWXg1/JFVafmDnw7hpJktQaXRlGllqouthGd9FIklSrrgwjKwkY7qKRJKleXRlGzj51oKm78rqLRpKk+nVdGBkZm+T6PZNNrRlxF40kSfXrujAyPDrBzOxPXkvkQBwVkSSpfl0XRppdvLphfW/NlUiSJOjCMNLM4tXenuDKC5yikSSpHboujAxt20xfb89+bb1HBBvW9xI0bog3fNErnaKRJKlNar0Ca6c6at0R+9aNbFjfy5UXvNzwIUlSIV0VRkbGJtmxa3y/Bax/N/tcwYokSVJXTdMcaCfNzOwcw6MThSqSJEldFUYOtpNmJZeHlyRJrdVVYeRgO2m8/4wkSeV0VRg50E6avt4e7z8jSVJBXbWAdX7HzPDoBHunZ9jY38fQts3upJEkqaCuCiPQCCSGD0mSOkdXTdNIkqTOYxiRJElFGUYkSVJRhhFJklSUYUSSJBVlGJEkSUUZRiRJUlFddZ2RkbFJL3gmSVKH6ZowMjI2yY5d4/vu2js5PcOOXeMABhJJkgrqmmma4dGJfUFk3szsHMOjE4UqkiRJ0EVhZO/0zIraJUlSe3RNGNnY37eidkmS1B5dE0aGtm2mr7dnv7a+3h6Gtm0uVJEkSYIuWsA6v0jV3TSSJHWWrgkj0Agkhg9JkjpL10zTSJKkzmQYkSRJRRlGJElSUYYRSZJUlGFEkiQVZRiRJElFGUYkSVJRhhFJklSUYUSSJBVlGJEkSUUZRiRJUlGGEUmSVJRhRJIkFWUYkSRJRRlGJElSUevqfPOI6AeuAU4HEvj1zPw/dZ7zYEbGJhkenWDv9Awb+/sY2raZ7VsGS5QiSZIWqDWMAB8GvpiZF0XEkcD6ms93QCNjk+zYNc7M7BwAk9Mz7Ng1DmAgkSSpsNqmaSLiWOAXgY8CZOYzmTld1/mWMjw6sS+IzJuZnWN4dKJEOZIkaYE614ycBEwBfxYRYxFxTUQcs/hFEXFZROyOiN1TU1O1FLJ3emZF7ZIkqX3qDCPrgDOBqzNzC/AU8O7FL8rMnZm5NTO3DgwM1FLIxv6+FbVLkqT2qTOMPAQ8lJm3V8efoRFO2m5o22b6env2a+vr7WFo2+YS5UiSpAVqCyOZ+V3gOxEx/xv/XODuus63lO1bBvngW17BYH8fAQz29/HBt7zCxauSJHWAunfT/CvgumonzQPApTWf76C2bxk0fEiS1IFqDSOZeSewtc5zSJKk1c0rsEqSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrqoPemiYgXLPWDmflY68uRJEndZqkb5e0BEgjgJcDj1eN+4EHgpNqrkyRJa95Bp2ky86TMfBnwP4ALMvP4zDwOOB+4qV0FSpKkta2ZNSNnZeYX5g8y86+AV9dXkiRJ6iZLTdPM2xsR7wH+ojp+K7C3vpIkSVI3aWZk5BJgALgB2FU9vqTOoiRJUvdYcmQkInqA383My9tUjyRJ6jJLjoxk5hzw2jbVIkmSulAza0bGIuJG4NPAU/ONmbmrtqpabGRskuHRCfZOz7Cxv4+hbZvZvmWwdFmSJInmwsjRwKPAOQvaksb6kY43MjbJjl3jzMzOATA5PcOOXeMABhJJkjrAsmEkMy9tRyF1GR6d2BdE5s3MzjE8OmEYkSSpAywbRiLiaOCdwMtpjJIAkJm/XmNdLbN3emZF7ZIkqb2a2dr7CeBFwDbgb4ATgB/WWVQrbezvW1G7JElqr2bCyMmZ+XvAU5n5ceA84BfqLat1hrZtpq+3Z7+2vt4ehrZtLlSRJElaqJkFrLPV39MRcTrwXeCF9ZXUWvPrQtxNI0lSZ2omjOyMiA3A7wE3As+rHq8a27cMGj4kSepQzeymuaZ6+DfAy+otR5IkdZtmdtPcD9wG3Arcmpl31V6VJEnqGs0sYD0N+C/AccBwRNwfETfUW5YkSeoWzYSRORqLWOeA54BHqj+SJEmHrZkFrD8AxoGrgI9k5qP1liRJkrpJMyMjlwBfAf4F8KmIeF9EnFtvWZIkqVs0s5vms8BnI+JU4I3AFcC7AC9hKkmSDtuyIyMRcX1E3Ad8GFgPvB3YUHdhkiSpOzSzZuSDwFhmzi37SkmSpBVqZs3I3cCOiNgJEBGnRMT59ZYlSZK6RTNh5M+AZ4BXV8eTwPtrq0iSJHWVZsLIz2TmH1LdMC8znwai1qokSVLXaCaMPBMRfUACRMTPAD+qtSpJktQ1mlnAeiXwReDEiLgOeA3wjjqLkiRJ3aOZ64zcHBF3AGfRmJ65PDO/X3tlkiSpKzQzTUNmPpqZ/z0zPw+8ICI+UnNdkiSpSxw0jETEz0XETRHxjYh4f0S8OCKuB75MY7uvJEnSYVtqZOQjwCeBXwamgDuB+4GTM/NDbahNkiR1gaXWjByVmR+rHk9ExOWZ+a421CRJkrrIUmHk6IjYwo+vKfKjhceZeUfdxUmSpLVvqTDyMHDVguPvLjhO4JxmThARPcBuYDIzvYy8JEnaz0HDSGae3aJzXA7cA/xUi95PkiStIU1t7T1UEXECcB5wTZ3nkSRJq1etYQT4I+BdwHMHe0FEXBYRuyNi99TUVM3lSJKkTlNbGImI84FHMnPPUq/LzJ2ZuTUztw4MDNRVjiRJ6lDLXg4+Is48QPMTwLcz89klfvQ1wJsj4k3A0cBPRcRfZObbDq1USZK0FjVzo7z/DJwJfJ3Gtt7TgbuAYyPin2fmTQf6oczcAewAiIjXAb9jEJEkSYs1M02zF9hSTaX8PLAFeAB4PfCHdRYnSZLWvmZGRn42M++aP8jMuyPi1Mx8ICKW+rl9MvOvgb8+pAolSdKa1kwYuSsirgY+VR3/CnB3RBwFzNZWmSRJ6grNTNO8A7gPuKL680DVNgu06sJokiSpSzUzMvJG4E8y8z8e4LknW1yPJEnqMs2MjFwA/L+I+EREnB8RzQQYSZKkpiwbRjLzUuBk4NPAJcD9EeHl3SVJUks0NcqRmbMR8Vc07tbbB2wHfqPOwlplZGyS4dEJ9k7PsLG/j6Ftm9m+ZbB0WZIkqbLsyEhEvDEiPgZ8E/hlGje9e1HNdbXEyNgkO3aNMzk9QwKT0zPs2DXOyNhk6dIkSVKlmTUjbwdGgM2Z+Y7M/MIyl4HvGMOjE8zMzu3XNjM7x/DoRKGKJEnSYstO02TmJQuPI+K1wCWZ+Zu1VdUie6dnVtQuSZLar6m79kbElogYjohvAf8WuLfWqlpkY3/fitolSVL7HTSMRMTPRsSVEXEv8MfAg0Bk5tmZ+cdtq/AwDG3bTF9vz35tfb09DG3bXKgiSZK02FLTNPcCtwLnZ+Z9ABHxW22pqkXmd824m0aSpM61VBh5C3AxcEtEfJHGvWmauzNeB9m+ZdDwIUlSBzvoNE1mjmTmxcCpwC007kvzwoi4OiJ+qV0FSpKkta2ZK7A+lZmfzMwLgBOAMeDf1F6ZJEnqCk3tppmXmY9n5s7MPLeugiRJUndZURiRJElqNcOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkooyjEiSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkooyjEiSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkoqqLYxExIkRcUtE3B0Rd0XE5XWdS5IkrV7ranzvZ4Hfzsw7IuL5wJ6IuDkz767xnJIkaZWpbWQkMx/OzDuqxz8E7gEG6zqfJElandqyZiQiNgFbgNsP8NxlEbE7InZPTU21oxxJktRBag8jEfE84Hrgisz8weLnM3NnZm7NzK0DAwN1lyNJkjpMrWEkInppBJHrMnNXneeSJEmrU527aQL4KHBPZl5V13kkSdLqVufIyGuAfwqcExF3Vn/eVOP5JEnSKlTb1t7M/J9A1PX+kiRpbfAKrJIkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkooyjEiSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkooyjEiSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkooyjEiSpKIMI5IkqSjDiCRJKsowIkmSijKMSJKkogwjkiSpKMOIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrKMCJJkopaV+ebR8QbgA8DPcA1mfkHdZ5vsZGxSYZHJ9g7PcPG/j6Gtm1m+5bBdpYgSZKWUVsYiYge4E+B1wMPAV+NiBsz8+66zrnQyNgkO3aNMzM7B8Dk9Aw7do0DGEgkSeogdU7TvAq4LzMfyMxngE8BF9Z4vv0Mj07sCyLzZmbnGB6daFcJkiSpCXWGkUHgOwuOH6ra9hMRl0XE7ojYPTU11bKT752eWVG7JEkqo/gC1szcmZlbM3PrwMBAy953Y3/fitolSVIZdYaRSeDEBccnVG1tMbRtM329Pfu19fX2MLRtc7tKkCRJTahzN81XgVMi4iQaIeRi4FdrPN9+5hepuptGkqTOVlsYycxnI+JfAqM0tvZem5l31XW+A9m+ZdDwIUlSh6v1OiOZ+QXgC3WeQ5IkrW7FF7BKkqTuZhiRJElFGUYkSVJRhhFJklSUYUSSJBVlGJEkSUUZRiRJUlGRmaVr2CcipoBv1/DWxwPfr+F9dWD2d/vY1+1jX7eX/d0+dfb1SzNz2RvPdVQYqUtE7M7MraXr6Bb2d/vY1+1jX7eX/d0+ndDXTtNIkqSiDCOSJKmobgkjO0sX0GXs7/axr9vHvm4v+7t9ivd1V6wZkSRJnatbRkYkSVKHMoxIkqSi1nwYiYg3RMRERNwXEe8uXc9qFxHXRsQjEfGNBW0viIibI+Kb1d8bqvaIiP9U9f3XI+LMcpWvPhFxYkTcEhF3R8RdEXF51W5/1yAijo6I/xsRX6v6+31V+0kRcXvVr/8tIo6s2o+qju+rnt9Usv7VKCJ6ImIsIj5fHdvXNYiIb0XEeETcGRG7q7aO+hxZ02EkInqAPwXeCJwGXBIRp5WtatX7GPCGRW3vBr6UmacAX6qOodHvp1R/LgOublONa8WzwG9n5mnAWcBvVv//2t/1+BFwTma+EjgDeENEnAX8e+BDmXky8Djwzur17wQer9o/VL1OK3M5cM+CY/u6Pmdn5hkLrifSUZ8jazqMAK8C7svMBzLzGeBTwIWFa1rVMvMrwGOLmi8EPl49/jiwfUH7n2fDbUB/RLy4PZWufpn5cGbeUT3+IY0P7UHs71pU/fZkddhb/UngHOAzVfvi/p7/d/gMcG5ERJvKXfUi4gTgPOCa6jiwr9upoz5H1noYGQS+s+D4oapNrfXTmflw9fi7wE9Xj+3/FqmGpbcAt2N/16aaNrgTeAS4GbgfmM7MZ6uXLOzTff1dPf8EcFx7K17V/gh4F/BcdXwc9nVdErgpIvZExGVVW0d9jqyr+wTqLpmZEeF+8RaKiOcB1wNXZOYPFn4htL9bKzPngDMioh+4ATi1cElrUkScDzySmXsi4nWl6+kCr83MyYh4IXBzRNy78MlO+BxZ6yMjk8CJC45PqNrUWt+bH8ar/n6karf/D1NE9NIIItdl5q6q2f6uWWZOA7cAf5/GMPX8F7eFfbqvv6vnjwUebXOpq9VrgDdHxLdoTJ+fA3wY+7oWmTlZ/f0IjZD9Kjrsc2Sth5GvAqdUK7SPBC4Gbixc01p0I/Br1eNfAz67oP3t1erss4AnFgwLahnVnPhHgXsy86oFT9nfNYiIgWpEhIjoA15PY53OLcBF1csW9/f8v8NFwJfTq0g2JTN3ZOYJmbmJxufylzPzrdjXLRcRx0TE8+cfA78EfIMO+xxZ81dgjYg30Zib7AGuzcwPFC5pVYuI/wq8jsYtp78HXAmMAH8JvAT4NvBPMvOx6pfpn9DYffM0cGlm7i5R92oUEa8FbgXG+fG8+u/SWDdif7dYRPwcjYV8PTS+qP1lZv5+RLyMxrf3FwBjwNsy80cRcTTwCRpreR4DLs7MB8pUv3pV0zS/k5nn29etV/XpDdXhOuCTmfmBiDiODvocWfNhRJIkdba1Pk0jSZI6nGFEkiQVZRiRJElFGUYkSVJRhhFJklSUYUTSYYuIueqOoN+IiE9HxPqq/cnlflaSDCOSWmGmuiPo6cAzwD8rXZCk1cMwIqnVbgVOXtgQEc+LiC9FxB0RMR4RF1btvx8RVyx43Qci4vKIeHFEfGXBaMs/aPN/g6Q28qJnkg5bRDyZmc+r7htyPfDFzLx6Ufv66kZ/xwO3AacALwV2ZeaZEXEE8E0a9814B3B0daXInupnf1jkP05S7bxrr6RW6IuIO6vHt9K4p85CAfy7iPhFGpe2H6RxC/NvRcSjEbGFxi3MxzLz0Yj4KnBtdaPAkcy8E0lrlmFEUivMZOYZSzz/VmAA+PnMnK3u1np09dw1NEZCXgRcC5CZX6mCy3nAxyLiqsz887qKl1SWa0YktcOxwCNVEDmbxvTMvBto3JTr7wGjABHxUuB7mfkRGmHlzDbXK6mNHBmR1A7XAZ+LiHFgN3Dv/BOZ+UxE3AJMZ+Zc1fw6YCgiZoEngbe3uV5JbeQCVklFVQtX7wD+cWZ+s3Q9ktrPaRpJxUTEacB9wJcMIlL3cmREkiQV5ciIJEkqyjAiSZKKMoxIkqSiDCOSJKkow4gkSSrq/wNQR4uazzIQzQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(1,1)\n",
    "ax.set_xlabel(\"Plays\")\n",
    "ax.set_ylabel(\"Avg Reward\")\n",
    "fig.set_size_inches(9,5)\n",
    "rewards = [0]\n",
    "for i in range(500):\n",
    "    p = softmax(record[:,1],tau=0.7)\n",
    "    choice = np.random.choice(np.arange(n),p=p)\n",
    "    r = get_reward(probs[choice])\n",
    "    record = update_record(record,choice,r)\n",
    "    mean_reward = ((i+1) * rewards[-1] + r)/(i+2)\n",
    "    rewards.append(mean_reward)\n",
    "ax.scatter(np.arange(len(rewards)),rewards)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContextBandit:\n",
    "    def __init__(self, arms=10):\n",
    "        self.arms = arms\n",
    "        self.init_distribution(arms)\n",
    "        self.update_state()\n",
    "        \n",
    "    def init_distribution(self, arms):\n",
    "        # Num states = Num Arms to keep things simple\n",
    "        self.bandit_matrix = np.random.rand(arms,arms)\n",
    "        #each row represents a state, each column an arm\n",
    "        \n",
    "    def reward(self, prob):\n",
    "        reward = 0\n",
    "        for i in range(self.arms):\n",
    "            if random.random() < prob:\n",
    "                reward += 1\n",
    "        return reward\n",
    "        \n",
    "    def get_state(self):\n",
    "        return self.state\n",
    "    \n",
    "    def update_state(self):\n",
    "        self.state = np.random.randint(0,self.arms)\n",
    "        \n",
    "    def get_reward(self,arm):\n",
    "        return self.reward(self.bandit_matrix[self.get_state()][arm])\n",
    "        \n",
    "    def choose_arm(self, arm):\n",
    "        reward = self.get_reward(arm)\n",
    "        self.update_state()\n",
    "        return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "arms = 10\n",
    "N, D_in, H, D_out = 1, arms, 100, arms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "env = ContextBandit(arms=10)\n",
    "state = env.get_state()\n",
    "reward = env.choose_arm(1)\n",
    "print(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    "    torch.nn.ReLU(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = ContextBandit(arms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(N, pos, val=1):\n",
    "    one_hot_vec = np.zeros(N)\n",
    "    one_hot_vec[pos] = val\n",
    "    return one_hot_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [],
   "source": [
    "def running_mean(x,N=50):\n",
    "    c = x.shape[0] - N\n",
    "    y = np.zeros(c)\n",
    "    conv = np.ones(N)\n",
    "    for i in range(c):\n",
    "        y[i] = (x[i:i+N] @ conv)/N\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(env, epochs=5000, learning_rate=1e-2):\n",
    "    cur_state = torch.Tensor(one_hot(arms,env.get_state())) #A\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    rewards = []\n",
    "    for i in range(epochs):\n",
    "        y_pred = model(cur_state) #B\n",
    "        av_softmax = softmax(y_pred.data.numpy(), tau=2.0) #C\n",
    "        av_softmax /= av_softmax.sum() #D\n",
    "        choice = np.random.choice(arms, p=av_softmax) #E\n",
    "        cur_reward = env.choose_arm(choice) #F\n",
    "        one_hot_reward = y_pred.data.numpy().copy() #G\n",
    "        one_hot_reward[choice] = cur_reward #H\n",
    "        reward = torch.Tensor(one_hot_reward)\n",
    "        rewards.append(cur_reward)\n",
    "        loss = loss_fn(y_pred, reward)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        cur_state = torch.Tensor(one_hot(arms,env.get_state())) #I\n",
    "    return np.array(rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [],
   "source": [
    "rewards = train(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x134095240>]"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VNX9x/H3yZ4QdgKySdgE2bcCKioqKgLihntbN9Ta1vVXFevW1gWq1rrVrVRbrQsttW4oKlQFN5R93xM2WcIWCNmT8/tj7kxmMpNkEiaZzMzn9Tx5uHPvyZ1zZ8h3zpx7zvcYay0iIhJd4sJdARERCT0FdxGRKKTgLiIShRTcRUSikIK7iEgUUnAXEYlCCu4iIlFIwV1EJAopuIuIRKGEcD1xmzZtbGZmZrieXkQkIi1atGivtTajpnJhC+6ZmZksXLgwXE8vIhKRjDFbgimnbhkRkSik4C4iEoUU3EVEopCCu4hIFFJwFxGJQgruIiJRSMFdRCQKKbiLBOHjFTuZu2Z3uKshErSwTWISiRRl5Zab3lgMQPa08WGujUhw1HIXqUHO4SLP9qItB8JYE5HgKbiL1GDk1Lme7Yte+IYLnv86jLURCY6Cu0gA89bn8PgnawMeW7L1YAPXRqT21OcuUsmMH7Zy939WAHDViZnhrYxIHanlLuLlYH6xJ7ADzFm9B4D7J/Rh4yPnePbnF5c2eN1EakPBXcTLl+tzfB4/5nTNbD+QT0J8HOcP6gBAnwc+oaSsvMHrJxIsBXcRx+HCEm59e6nncUpiHAfzSwCYMKA9ACf2aOM5/spXWbz2bTbW2gatp0gwFNxFHH/4YLVne8YNI2mSVHFLqlubdAAy0pM9+6Z+vJYH3lvFf5fsaLhKigQpqOBujLndGLPKGLPSGPOWMSal0vGrjTE5xpilzs/k+qmuSP3p0Tbds/2TzFYkJ1T8eaQmxQNwWu+2vHL1MJ/fu+NfyxqmgiK1UGNwN8Z0BG4Bhllr+wHxwGUBis6w1g5yfqaHuJ4iDSouzvBjbiEAg49tQUpivOdY/44t/MqXqv9dGplgu2USgFRjTAKQBvxYf1USaXjb9ucz9WPXzdMLh3QE4JTjXGsQP3XpIJ+yLdMS/X7/ncXqmpHGpcbgbq3dATwBbAV2ArnW2k8DFL3IGLPcGDPTGNM5xPUUqVcnP/a5Z/vJS1zB/OWfDeXdX51El9ZNfMomxPv/2WTtOxLS+mROmcUvXl9U5fHSsnI+WbWLJVuVDkECC6ZbpiVwHtAV6AA0Mcb8tFKxD4BMa+0A4DPgH1Wc6wZjzEJjzMKcnJxARUQajZTEeAZ19u+C8TZxoGto5AtfbArZ8xaWlAEwe9WuKkfi/N+/l3Hj64u44PlvKC0r53fvr2Lb/vyQ1UEiXzDdMmOALGttjrW2BHgHONG7gLV2n7XWnV1pOjA00ImstS9ba4dZa4dlZGQcTb1F6sXHt55cq/JPX1bRZROqIZFbvYJ0XlHgyVLvLa3oGe1x78f8/Ztsn28fIsEE963ASGNMmjHGAGcAa7wLGGPaez2cWPm4SGO2/UBFMD2+fbOgfufpywZx4yndcP1JOPvmbghJfXY5N3IB9uYV+x13t+wDicUx989/sZF/LdwW7mo0OsH0uS8AZgKLgRXO77xsjPmDMWaiU+wWZ6jkMlwja66up/qKhETmlFkMe3gOAKP+WPsW73mDOnLPuOMBeP7KIUDogvvXm/Z6tvfmFfkdn/DsV1X+bm5BSUjqAFBcWk55eeP+sCgrtzw2ex13zVweluffm1fEc//bQFkjfJ2CGi1jrX3QWtvbWtvPWvsza22RtfYBa+37zvF7rLV9rbUDrbWnWWsDp9MTaQTcrdvKgfOdX54YqHiNzuzTzjkvfLDs6AeSvfTlZs/2xS9+y0MfruaiF77xtNi3VtO3/vm6PVUey9p7xOdbQXWstRx338fc997KIGsdHll7K25k7z/i+y3nSBVdWqE07OE5PPHpehZm76/356otzVCVmDNnTUUA3OcE+FZNkhhybMs6nS/Ra/TMzW8tOaqukUC/+7evsli05QC9758NQK92TQGYf9dpfmVvnxF4QtU3m/Zy2hNf+OSmr07Xez4C4M0FW3lv6Q6fBUsakz2HKj6s9hwuJK+olJKych6ZtZq+D37C91m+QXfKf5ZX+wEI8P6yH8mcMqtWcxfc35gKS8qq7TZrSAruEnPeXVoxJv3Jz9YD/q2+2hrdq2KAwKGCurcYcwJ0w3iz1rJiRy4DO7egc6s0zu7r+tYw5Zze1f7ea99s8WzXFLQqf8Dc+vZSfvLInGp/p6HlFZWyeOsBth8s8OybPj+Lfg9+wvBH5vDX+VkAXPLSt3yftZ+1uw6Rc7iIt3/YxjWv/sC2/fls3HM44LlveWsJAFu8viHNWr6TO/9d9UzkG15fxL68InrfP5ve989uFPc+lM9dYo/X392mnLyQnPLv1wwnc8osAF6at4m7xrqCbXm5JS7OVPerPhZvcS0EMnlUV6Z/leV33N2izne6HB46vx+92jVl8qiufLd5H9l7A4+3P+KVoji/pIxm8XFYa7n61R94+Px+dG6V5jm+KSfwOcY8+SVz7jg16GupT/0e/MRv38xF2wE4kO973+GSl771K+seWVTdmrgLs/fTPcOVkuJXb7rW0N1/pJi/Xf2TgOWHPlzxAfjGgq0cLiwlPSWB0cdl+Ly+DUUtd4kpR4pKmbViJwBxBvY43Q1Xh2BRjqkX9gfgeWfM+1cb9tLttx/59AvX5Bf/dE1c+unILvTt4Bq507pJkl+5Y51g0bZpCnec1YuE+DgyWzche18+mVNm+d3gm7+h4ibtjgOu1u4nq3bz5focvyGU71aRCG3jntB8EB6tULaKA3WhpCe72rxLt+X6Pd/ctXs4VFhS5e+63ffuSv44ey33v7uS3/53RZXl6pOCu8QUd2AHKLew2WmlTnTytB+NE7u39nl810zX1/jl22u/LF9mmyb0cYZl7jtSzJ1n9/I5fsdZx/n9jjvogO9C3pWD4TlPz+fL9Tk+31r+t3Y31lqKSss83REPntuHlMTwhghrLdPnb2ZnbkX3y89f+d6nzOBjA080W/vQWDq3Sq32/FdOX+D3Qei+Zve/y7fn+hx/Zo5rVJS7n31ol+rv1ewM8iZ2qCm4S8xYv/uwZ8jcny8d6HNscA0zUYPhnabgSFGpJ/HYW99vrdP50lNcLch7zunNTad29+y/aXR3+nZo7lc+Ma7iz/mf31X0sR8q9L8HcMeMpTz+yTrP42v/vpCHPlxDr/tme0b8TBraiTV/GEvW1HHcckZPjKHBh/w9PXcDD89awwlT/+fZ5/0tBFxdX69fN9zvd1MS45l352k8ekF/z77lvzvLp8yiLQd47n8bAdiZW0DmlFmeuQUfLHM1BErLfe9RZDR1pX1es/MQ4MpFdO1JXT3Hu7XxTVcRrm88Cu4SM8768zzP9vCuvq1s78lIofDUnPWe7fIgB124b3Se1MNVt3Od1Aaje7X16be/e2zgm6d3ju3lWVSkqLSiy8A9omR8/4q5hvsC3EB+5euKPv5mKQk0TUnEGIMxhuapiVjrGpHSkJ6aUzF34P1Kw0zd32zeuH4kmV4frKN6tOE05wa3MYYrRhzrOdYsxT/p2/vLXN1Q3h8gUDFUdrvTjfXm9SMAVx7/zCmzuPrVHwBIS4rngXP7kD1tPNnTxtPdSR3d+5imnnOF4wargrvEpBapFX/kI7q2Ctl5xxzfFsAzWgPg++z9QbV4P1juCl77nJbjkGNbkj1tPL2cIOEOHlVpk57Mc1e4JlSt+vGQZ/+ZzoeaO9tlMCq39v823zX2vnIAbEi3vLXEc9Ma4LpRXcmeNp705AQ6t0qjY4tUfn1aD/45eQSvXuPbkn/4/H5cNKQT4JsyAlzDYL3P6y1zyizP6lwdWwTu4jm9dzufx5+t3g3A2l2HPa/5xOe+pqzcsvtQYa3uwRwNBXeJSWlJ8Xx48yj6dmjGK1WMfqiLxycNDLjf3TqsSlFpmWeMuvdX/LrafqCAotIyrvv7D559XVo3IXvaeG45vYdnn/vbQU2ap/nf1K0Lay23vb2EP84++nmOFw3t5PP46ymn85tK9ybcfjqyC3+6xPXenDeoI9nTxnuufcWOXL/yXVr7j25p3zxwcG+e6v9twM2dCnrFjly6//YjRjw6l9Oe+KLK8qGk4C5R62B+MT84Mwf/8vlGn2PGGPp1bM6sW06mSXLoRgRX9Ye+K9f1FX9TTh6ZU2b53CAE35b2pEpBq7bc+eanz89i7tqKCTvulaZuG3Mcj00awFOXDuLZywdX+23AbcaNIz3bPx4s8DsebEbKrvd8xLtLf+SFLzYFNd7eGLjl9B5cXOk12fDIOUE9X3WevXwwaUnxFJb41uPVa37Cln3+15OUEMeVXl08VblrrOsDZszxbfnHtf73AgBun7E04P5QUnCXiGatJXPKrIAtwQue/4aLX/yWVT/m+tw8/Pw3o+utPnFxxjMkEiqGWJY7fa5/+tRVj3eX+PYfL8p2jW45vXfbWo2LD2TiwA40T02sMs9MXJzhkmGdOX9wRTfNoxf0Z2Cn5nx6+ylAxVBLN+++6pfnVaRHyM0vIXPKLE5+7PMqh1BWxbs/PZCCkjKshbTkBIZlVoxIiY8zPrOCj0Z+ccW9iUcu6Ef2tPGc1qstj1zQz6fcb8f1dsr099n/7OWD/c555YguTBzYgccnDeSkSiOo3Bpi3V0Fd4lo7j/OF77YxLJtFUMOrbWevs3xz1Qk2lr38Fi6VhrNEGqXDz+Wds1cIyrun9CHtKR4z825j1bsAqCpMxJmV24hf5y9lpfmucbGv/SzgNmya6V5aiKHCkt8WtPum4FVuWLEsbz361Ec164p2dPGMy9AagO3/ziThd5csJWBf6hYt+e2GUvJnDKrytm+le87pCXHByzn5u67LiwpY83OitmkC+8dU+3v1VXnlhUfaFeO6EL2tPF8eedoNj06jhtOqRit9PyVQxjWpSUDOjUP2K3VPDWRZy4fTMsmSSTEx5E9bTzPXeH7IVB51E59UHCXiOa+2QVw3l++9my/9u2WQMVJTqg+oITKgt+OIXvaeOLjDPnFZbz6dbbP7NH73l1J5pRZjJw6lxe+2OQZfheKFmkzZ2TLxytdHyRZU8dxYvc2R33eD28eBcDholKKSsuqnJzz5fo9PqN13A7m+wb9FqlV9+N/tGKn571NTojnF15DQVsGmNRVV7/xmi/QKsB5u7RuQnylb1Lj+rdn5k0n8v6vRwX9PBMGdGCgM9z2l6O7Bxy1E2oK7hLR5qzZ7fPYnQjswfdXhaM61RrdQDfSCop9A2uohnn2bJfu2e513+wqy90+Y1nA4wudiVXuLo6CkjKWbz/o90FQXFrOL99Y7Hl8wyndaNs0mXH9j+G1Kvqw6+raUa6b11efmEm/jv5zB0Lprz8byvDMVlwVgtnQwVBwl4hVXOp/Q27ow3N8Vi8a0KniD9Z73HFDCnb0yw2ndAvJ843qefSt9EBq+62n8vtzo7Mm7Lpdrkk9D324monPfc3IR12ZKr/ZtJcV23P9Jg3Fxxni4gzPXznUs2h5qKQlJZA9bTy/m9g3pOcNpG2zFP71ixNo1yyl3p8LFNwlgu32SvfqPT3fO6nU69dW9DW/FmAWY0N44Nw+QZXzXhHqaFQ1HjsUvr3n9ID7v5lyOn9xxti7bd0feDz3bWN6+jw+kF/C3rwirvjrAs597iv6PFDx/n155+ijq3AMU3CXiJI5ZRbnO33rS7xuoP7qtB5+Ze8e25tmqQn8bGQX/vvLE2nbtGFaTLXxxW9G09MZovjkJYNqKB2cFl5j0uf+X2izOHqP9fbOO9OhRSrjB7TnV6dV9I1f/9qigOcIlCFx2MP+KYUnDGjvk9JBakfBPcZ8vq4iq11ZuWXD7sNkTpkVstS3DWGpE9TdebfvONM/iRZA++YpGGN46Px+DK7jQhyhMsRJbvXZ7ad4Au5b148ks00TPrvjVLKnjSclMTQ3e5MSKv6s3SlrQ2nenafx3q9OYu1D5/Cfm07k+9+e4Tl259m9mXOHazhl1t4jTHh2PlAxDv6Ebq6hgfeNP77G57lkWOdQVz2mKJ97lPk+az/z1ucEnKm3L6+Ia5x8GG9OHsG97670DBc8409fBjWZJRwOHClmwrNf+XTDeE8X/+nILgB8dfdpPuuhHtO88bTU/zl5BPnFZbRJdw2RbIjX2p1nJtSObZ3Gsc4MzkAZEb1HgqzccYhNOXkccIZHju13DACTT+7GlSO6cPZT86pcNrCqbI8SHLXcG7nXvs3m6417ayzndslL3/Lc5xt9+m9z80uY+tEavtm0z7PviukLfHJcdKgiEE6fv5k3F9Qtq2GoDH7oM3YcLKC0ivws7iFsnVqmMfu2kz37O1QxXTwc0pISPIG9IbjGVg+puWA9qHzj9Yw/fclh5yZ3f68b3KlJ8ZxaxQ3Si4d28uRVl7qJ+eBureXumct9WoXhkF9c6regb3FpOQ+8t4orpy+o9fk+WVUxRPCFLzfx0rzN3Ox0YwTyY26hX+a68nLLw7PWNPhiA//6YRuZU2axbX9+jQm3Xq2UF6b3Mc082x1bNp7gHktSk/y7l3Kd1ZEqp2c4u+8xAc/x+MUDQ56pM9bE/Efjgqz9zFi4jRkLt7lmpK3PoUVqomfCQUNxjxCYNLQT5dZ6Eg65lZSVeya4PDt3AyXllmtPymTQHz4DXKvHeA8BfOjD1Zx6XBt6tG3q+Upckx+yDzDcyZBYXm6Z+vEaz7FPV+3ipB5tqs3DcqSolLlr9zAxyGRUAN9s3EtKUrzP4tR3/ceVc33JtoM+/ceBtG3m3xpurN1LsSIpIY6Vvz+b9bsPc+Hz3wCu2atQkQvdzXvYZrOUBKZdNMBn+KrUXUwH97Jyyx1eCXy63TMLd0OxIQOE9wru7nUgK3vjuy2ccXw7OrZI5U/Oos7PzK3IzZFX5L8gw968YrpnWGYs3Oazv6r1OQu8lg3r9tuPfI7d4IxRrup12bY/37Nc2y1vLSF72njPN4FALbCVO3LJOVzENU7WQvd5vWcx/u2rLAZUM7GkWUpCWNamlJqlJycwsJN/A6lpgMbBxUM78e9F2/nw5pM9ffly9GI6uD/x6TrPajkADbzIjMdlf/2uxjK/+2A1v/tgNbee0bPGsm5/+XwjW/f7tqLvn9CH60Z1Ja+olLd/2MZkZ4be9K+yWLvzUJV9oDVxr0rj1vWeWfTr0JwVO3JZ9uBZfl/HJzz7FZUdLizxtPTAFbwPO3nFX/7ZUMYc346X529m2sdrOalHa96YPNLvHNJ4xMcZfj+xr89s4UAf9I9fPJDHLw6cKlnqLqb73F9wFjIOpKZ0pKEw+R8L+Wz1bs86noG0r3Sj8+m51WfS83Z8+2aeLIRu7pSl0y4aQPa08dw3oQ+TT3bNjJz6cc05tr3X6fS2dtdhn8fWVuTJHvh7V3Kp3YcK2ZdXxKIt+/1+P7+4lHHPzGez103epPg4DjvP1zQlkbg4w/mDOtK5VSqPnN/f7xzS+Fx1Yia/C3ISl4RWTAd3t+PbN/Pb90P2gQAlQ+dwYQlz1uzm+tcWBjzeq51rqvyUcwIvqVads/u6VobZnHOE3YeKfI4FGkvdIs0/iVFqYjwTBrT364Z5sYoPxCc/Wx9wv9u+vCJGPDqXoQ/P4aIXvvU73ueBT9i23zdP+Ny1e1jufEC4syge0zyF+XedTmY9Z3aU0Ln6pK48MKGPJ52wNIyY7pbp39HVbfDRLaPoeo9vH/PhKlqoofLrN/1Hriy6bwy5BSUUlpTz4fIfWbf7MKlVTGx5+Px+tElPZnSvDJ+AvWjLfvp2aE7v+2f7JNWq7h6C9+9nTpnFsgfOoqCkzLO4wz+uHc5Vzorzz3+xibsqreHpvRr911NO56Rp/kuxVZUGtibTnG8ToZrgI+HhTtAlDSemW+7rdh9mQKfmGGOYck5v7ht/PN/d45pt576BWFlufgnfbtrH1xv3+mXfq8nWffm8s9h1w/TL9Tl+x1unJ9MtI50+HZpx65iePD5pAGf2acfbN/j3Lf90ZBfG9jvGL+gN7dLqqAOhO0e3OyCfelyGJ/90m/QkTv/TFyzZeoBFW/ZTUFzGPK9r6dgilb4dXN+EvPO9VLUCfLMU//bFnDv8p8xXXlFeRKoX0y334tJylm93fe1354v27mtfv/swZ/15HiO6tmLGjSdgrfVZnABqN6rmiunfsf1AAQMCjCJY+9BYn8fJCfFc7Ey/HtmtNdnTxvOvhdu4a+Zyn9Xcg1HTQg1V8Z705J51uDevmL15xVzgdeOzsnd+eSKfrtrNhAHt6dk2nRteX8RNXilc3T749Sh6tkun9/2u9LBPXzaIiQM7YIxh7UNjPfuBo16dSCTWxHRwDyTBa7GEs5xV4xdk7WfF9lzOfc5/hEcwsvYe4bPVu9h+wNWn/Jt/L/MrE0xr+5JhnRnXv31QM/fuHtubOWt288zlg4PKEvjDvWN49KM1Pst/fXJb3fpIkxPiPSvUBLqf0blVKrNvPcVvzPyoHm08oym8X4+//nxYneohEstiNrgv2OxqlQY7Jdx7Qk9t3TZjqc8ScEu9ttukJ9GnQ/CTNoKdkn3T6O7cNLp7zQUdGU2TefKSgZ7gfmL31n4r0Dx7+eBqZ7kOz2zlt6/yqjlZU8cBvkPi7hrbi8dmr/PJZgjwz+tGsHjrAc7s0y7o6xARl5jtc1+Q5RqO517b0ttTl/qnXvXuovBW1Y3CotIy3lu6A2utT2D3dnbfdnx0y8k8f2V4coBUZoyhe4arb/u+8f7D184d2IHNj46r8vefCbBYcJNKU9GNMX5jnX85uodnSTpvo3q24ZZajOsXkQpBNQONMbcDkwELrACusdYWeh1PBl4DhgL7gEuttdkhr20IuQNJoOARKNNdZSO6tmJB1n72HynyW3vRO2NhUYDVgtzuG9+Htg20Kkuw7jy7F/f+d2WVXTlxcYbLh3fmre99Z71+ePOogFkY3YG8XbNkPv/N6JDXV0QCqzG4G2M6ArcAfay1BcaYfwGXAX/3KnYdcMBa28MYcxnwR+DSeqhvyLgbj5NP9h+i5T2lvapAlnO4iAVZ+z0zKN3KK01zvWvm8irr0Binzo/t156z+x5TbdKmqRcO4NTj2tK5VSrjn3Hdh3CPkAlEuV5EGl6wfe4JQKoxpgRIA36sdPw84HfO9kzgOWOMsZXTDDYij812zdysahXyH+4dw/wNOVw4pBNDjm3JnV5BumlKRZKu7QcKPAtB/P6DVX7BPpAl958ZVLlwCSYb39h+x/h8kCmDn0jjUmNwt9buMMY8AWwFCoBPrbWfVirWEdjmlC81xuQCrYHgE5E3MhlNk7lwSCcALh7WmYuHdeaEqXPZmVtIWlKCJ0XwzW8t4cw+7UhJjOfVr7NrPO/gY1vQskmS343GSBQXZ9QqF2mkaryhaoxpiatl3hXoADQxxvy0Lk9mjLnBGLPQGLMwJ8d/Ek9DGfLQZ3X6vTevH8n9E/qQ0TSZtKSKz8V/fJPtV/Yf1w7HuzHr7m++/uTQrHAvIlKdYEbLjAGyrLU51toS4B3gxEpldgCdAYwxCUBzXDdWfVhrX7bWDrPWDsvIqFv2wVBwj3A5b1DweccBurZpwnXONOoxx7f17A+UcOvU4zIY2bW1z+9mTxvPuP71s/SZiIi3YIL7VmCkMSbNuDpWzwAqD/p+H7jK2Z4E/K+x9re/6zVJp/LKR7VRuY95XoB0Aq9UWiVIRKSh1BjcrbULcN0kXYxrGGQc8LIx5g/GmIlOsb8BrY0xG4E7gCn1VN+j9vsPKnJLu2eM1tWLP60Yn+6dPMstNSmexycN4O/XKMiLSMMKarSMtfZB4MFKux/wOl4IXBzCetUb79WGRvVoU03Jmo3t157nrhjsl+GxX8eKYYHu/DAiIg0pptIPFJWWUVjimlR06bDO/MYra2FdVc6XnjV1nIYFikjYxVT6gWleNz7/OGlASHKEn+WV92T6z4cpsItIoxBTwX3Vj4dqLlRL3qsYjVGCKxFpJGKqW6Y+2tRNUxJ58pKBnNj96PrvRURCKaZa7t2dZeNuPDW0E4kuHNIpYNIsEZFwiZngvv1APm8u2ArA3WfXftFpEZFIEjPB/e9eeV+0ZJuIRLuYCe7FZVXnVRcRiTYxEdz/8vlGXvt2S7irISLSYGIiuD/+yTrPdnvd+BSRGBATwd3bCd1a11xIRCTCRX1wzy0o8Xn86IX9w1QTEZGGE/XBfVNOnmf7retHhiTlgIhIYxf1wb2k1DVKpvcxTTmhu7pkRCQ2RH1wv+/dlQDcceZxYa6JiEjDifrgvmGPq1vmUGHdV10SEYk0UR/c3bzXPBURiXZRH9wT412pBlqkJYW5JiIiDSfqg3tJWaNcp1tEpF5FdXA/mF8MwPgB7cNcExGRhhXVwX30E18AMGv5zvBWRESkgUV1cE+Ic11e0+SYWnBKRCS6g/vPRnYB4Ku7Tw9zTUREGlZUB/fvNu8DoLnXItYiIrEgqoP7t05wFxGJNVEb3K3VEEgRiV1RG9yfmrMh3FUQEQmbqA3uq3ceAuAnmS3DXBMRkYYXtcF9QMfmALx+3Ygw10REpOFFbXA/WFBCWlK8FucQkZgUtcE9t6CE5qkaAikisanG4G6M6WWMWer1c8gYc1ulMqONMbleZR6ovyoH52B+sTJBikjMqnFevrV2HTAIwBgTD+wA/hug6Hxr7YTQVq/uDuSX0FKTl0QkRtW2W+YMYJO1dkt9VCaUDqlbRkRiWG2D+2XAW1UcO8EYs8wY87Expu9R1uuo5ReXkZqkm6kiEpuCDu7GmCRgIvDvAIcXA12stQOBZ4F3qzjHDcaYhcaYhTk5OXWpb9AKSspIU3AXkRhVm5b7OcBia+3uygestYestXnO9kdAojGmTYByL1trh1lrh2VkZNS50sHILy4lLUmpfkUkNtUmuF9OFV0yxphjjDHG2R7unDdsWbsKissoLCln3a7D4aqCiEhYBdW0NcY0Ac4EbvTa9wsAa+2LwCTgJmNMKVAAXGbDmLlr5Y+5AHy5vn67fkQ64WHeAAAMZklEQVREGquggru19gjQutK+F722nwOeC23V6i45wfWF5PcTw35fV0QkLKJyhur+I66Fsft1bBbmmoiIhEdUBvf84jIAmmjtVBGJUdEd3DVaRkRiVJQG91IATWISkZgVlcE9r8gV3NPVLSMiMSoqg/tjs9cBKJe7iMSsqAzuIiKxLuqCe2lZebirICISdlEX3N397bee0TPMNRERCZ+oC+6HClzBvXOrtDDXREQkfKIuuOcWlADQLEUjZUQkdkVdcJ+3wZUsTKswiUgsi7rg/vgnrmGQzRTcRSSGRV1wd2udnhTuKoiIhE1UBvcWaYm0bZoS7mqIiIRNVAV3ay3GwM9Hdgl3VUREwiqqgnt+cRnWKtWviEhUBfcj7oRhGgYpIjEuqoL7J6t3A/DFOq2dKiKxLaqCexMnf/spPduEuSYiIuEVVcE9zQnuQ7u0CnNNRETCK6qC+z5nYeyWTTSBSURiW1QF9+nzswBok54c5pqIiIRXVAX3rL1HAEiMj6rLEhGptaiJgiVapENExCNqgvu+vOJwV0FEpNGImuBeZm24qyAi0mhETXAvKC4DYPKormGuiYhI+EVNcP/znPUAJOhmqohI9AT3L52UA/vyisJcExGR8Iua4J7nJA27aXT3MNdERCT8oia4n3pcBgDdMtLDXBMRkfCrMbgbY3oZY5Z6/RwyxtxWqYwxxjxjjNlojFlujBlSf1UO7Mv1ygQpIuJWY+Jza+06YBCAMSYe2AH8t1Kxc4Cezs8I4AXnXxERCYPadsucAWyy1m6ptP884DXr8h3QwhjTPiQ1rIXEeNPQTyki0ijVNrhfBrwVYH9HYJvX4+3OvgZ1+fBjG/opRUQapaCDuzEmCZgI/LuuT2aMucEYs9AYszAnJ3R95O8v+xGA7zbvC9k5RUQiWW1a7ucAi621uwMc2wF09nrcydnnw1r7srV2mLV2WEZGRu1qWo21Ow8BMOTYliE7p4hIJKtNcL+cwF0yAO8DP3dGzYwEcq21O4+6dkHq0joNgF+f3qOhnlJEpFGrcbQMgDGmCXAmcKPXvl8AWGtfBD4CxgEbgXzgmpDXtBp5Ra68MunJQV2OiEjUCyoaWmuPAK0r7XvRa9sCvwpt1YJ3xJmd2kTBXUQEiJIZqnlFpaQkxmkFJhERR1REw8OFpeqSERHxEiXBvYSmKYnhroaISKMRFcE9t6CEZqkK7iIiblER3A8VlNBcwV1ExCMqgnuugruIiI+oCO6HCktplqIbqiIiblER3POKSklXcBcR8Yj44F5WbikuLSc1MT7cVRERaTQiPrgXlrhSDyi4i4hUiJ7gnqTgLiLiFvHBvcAJ7ilquYuIeER8cC9UcBcR8RPxwT23wJURMknrp4qIeER8cF+5IxdwTWQSERGXiA/ui7YcAGBQZy2xJyLiFvHBfWH2fgDapCeFuSYiIo1HxAf3cwd1ICkhjtbpyeGuiohIoxHxwT2/qIwmGuMuIuIj4oP7keJS0pKUV0ZExFvEB/f8ojKaJKvlLiLiLeKbvLNX7Qp3FUREGp2Ib7mLiIi/iG+5t0lP5sw+7cJdDRGRRiXiW+5HikpJV5+7iIiPiA7upWXlFJSUkZ6s9VNFRLxFdHA/UuzKCKnRMiIiviI6uK/deQiA+DhlhBQR8RbRwf3Sl78D4O3vt4W5JiIijUtEB/cBnZoD8Mo1PwlzTUREGpeIDu7Lt7tyuXdskRrmmoiINC4RHdxFRCSwoIK7MaaFMWamMWatMWaNMeaESsdHG2NyjTFLnZ8H6qe6IiISjGBnqD4NzLbWTjLGJAFpAcrMt9ZOCF3VqvfjwYKGeioRkYhTY3A3xjQHTgGuBrDWFgPF9Vutmu0/EvYqiIg0WsF0y3QFcoBXjTFLjDHTjTFNApQ7wRizzBjzsTGmb6ATGWNuMMYsNMYszMnJOZp6szO3EIDHJw04qvOIiESjYIJ7AjAEeMFaOxg4AkypVGYx0MVaOxB4Fng30ImstS9ba4dZa4dlZGQcRbXhxtcXAlBYUnZU5xERiUbBBPftwHZr7QLn8Uxcwd7DWnvIWpvnbH8EJBpj2oS0ppV0aunq9h/dq219Po2ISESqMbhba3cB24wxvZxdZwCrvcsYY44xxhhne7hz3n0hrquPrfvzATimeUp9Po2ISEQKdrTMzcAbzkiZzcA1xphfAFhrXwQmATcZY0qBAuAya62tjwpXlqC8MiIifoIK7tbapcCwSrtf9Dr+HPBcCOtVoxO6taas3OJ8YRARES8RO0O1oKSMxAQFdhGRQCJ2mb2l2w6GuwoiIo1WRLbcF2bvD3cVREQatYgM7t9scg3EUTZIEZHAIjK4u4P6a9cND3NNREQap4gM7gXOrNSmyRF7y0BEpF5FZHB3pxxISdLC2CIigURkcC8odgX31EQFdxGRQCIzuJeUkRBnSIyPyOqLiNS7iIyOhSXlarWLiFQjIu9IvvJ1VrirICLSqEVky11ERKoXkS33/h2b0yY9KdzVEBFptCIuuB8pKmXFjlzapCeHuyoiIo1WxHXLbNiTB8DevKIw10REpPGKuOCuxTlERGoWccH9r/M3AzBhQPsw10REpPGKuOB+yxk9adUkiUcu6B/uqoiINFoRd0O1e0Y6i+8/M9zVEBFp1CKu5S4iIjVTcBcRiUIK7iIiUUjBXUQkCim4i4hEIQV3EZEopOAuIhKFFNxFRKKQsdaG54mNyQG21PHX2wB7Q1idSBTrr4GuX9cfq9ffxVqbUVOhsAX3o2GMWWitHRbueoRTrL8Gun5dfyxffzDULSMiEoUU3EVEolCkBveXw12BRiDWXwNdf2yL9euvUUT2uYuISPUiteUuIiLViLjgbowZa4xZZ4zZaIyZEu76hIoxprMx5nNjzGpjzCpjzK3O/lbGmM+MMRucf1s6+40x5hnndVhujBnida6rnPIbjDFXheua6sIYE2+MWWKM+dB53NUYs8C5zhnGmCRnf7LzeKNzPNPrHPc4+9cZY84Oz5XUnjGmhTFmpjFmrTFmjTHmhFh6/40xtzv/91caY94yxqTE0vsfctbaiPkB4oFNQDcgCVgG9Al3vUJ0be2BIc52U2A90Ad4DJji7J8C/NHZHgd8DBhgJLDA2d8K2Oz829LZbhnu66vF63AH8CbwofP4X8BlzvaLwE3O9i+BF53ty4AZznYf5/9FMtDV+f8SH+7rCvLa/wFMdraTgBax8v4DHYEsINXrfb86lt7/UP9EWst9OLDRWrvZWlsMvA2cF+Y6hYS1dqe1drGzfRhYg+s//Hm4/uhx/j3f2T4PeM26fAe0MMa0B84GPrPW7rfWHgA+A8Y24KXUmTGmEzAemO48NsDpwEynSOXrd78uM4EznPLnAW9ba4ustVnARlz/bxo1Y0xz4BTgbwDW2mJr7UFi6P3HtTJcqjEmAUgDdhIj7399iLTg3hHY5vV4u7MvqjhfMQcDC4B21tqdzqFdQDtnu6rXIpJfo6eAu4By53Fr4KC1ttR57H0tnut0juc65SP1+rsCOcCrTrfUdGNME2Lk/bfW7gCeALbiCuq5wCJi5/0PuUgL7lHPGJMO/Ae4zVp7yPuYdX3vjMrhTcaYCcAea+2icNclTBKAIcAL1trBwBFc3TAeUf7+t8TV6u4KdACaEDnfOBqlSAvuO4DOXo87OfuigjEmEVdgf8Na+46ze7fzdRvn3z3O/qpei0h9jU4CJhpjsnF1t50OPI2ru8G9kLv3tXiu0zneHNhH5F7/dmC7tXaB83gmrmAfK+//GCDLWptjrS0B3sH1fyJW3v+Qi7Tg/gPQ07mDnoTrRsr7Ya5TSDj9hX8D1lhrn/Q69D7gHvFwFfCe1/6fO6MmRgK5ztf3T4CzjDEtndbQWc6+Rs1ae4+1tpO1NhPX+/o/a+2VwOfAJKdY5et3vy6TnPLW2X+ZM5qiK9AT+L6BLqPOrLW7gG3GmF7OrjOA1cTI+4+rO2akMSbN+VtwX39MvP/1Itx3dGv7g2uUwHpcd8HvDXd9Qnhdo3B95V4OLHV+xuHqR5wLbADmAK2c8gb4i/M6rACGeZ3rWlw3kjYC14T72urwWoymYrRMN1x/nBuBfwPJzv4U5/FG53g3r9+/13ld1gHnhPt6anHdg4CFzv+Bd3GNdomZ9x/4PbAWWAm8jmvES8y8/6H+0QxVEZEoFGndMiIiEgQFdxGRKKTgLiIShRTcRUSikIK7iEgUUnAXEYlCCu4iIlFIwV1EJAr9P91bRrtWiQKeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(running_mean(rewards,N=500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeprl]",
   "language": "python",
   "name": "conda-env-deeprl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
